From dcd345f6d3d47ff92e8acb32c9c7c499ed69778a Mon Sep 17 00:00:00 2001 From: LoGin Date: Thu, 17 Oct 2024 11:38:40 +0800 Subject: [PATCH 01/30] =?UTF-8?q?feat:=20=E6=B7=BB=E5=8A=A0SYS=5FMKDIRAT?= =?UTF-8?q?=E7=B3=BB=E7=BB=9F=E8=B0=83=E7=94=A8=20(#986)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 将内核原本实现的do_mkdir_at暴露出来,实现SYS_MKDIRAT Signed-off-by: longjin --- kernel/src/filesystem/vfs/core.rs | 3 ++- kernel/src/filesystem/vfs/syscall.rs | 8 ++++++++ kernel/src/syscall/mod.rs | 7 +++++++ 3 files changed, 17 insertions(+), 1 deletion(-) diff --git a/kernel/src/filesystem/vfs/core.rs b/kernel/src/filesystem/vfs/core.rs index f52470f6a..e519417bd 100644 --- a/kernel/src/filesystem/vfs/core.rs +++ b/kernel/src/filesystem/vfs/core.rs @@ -172,7 +172,8 @@ pub fn do_mkdir_at( user_path_at(&ProcessManager::current_pcb(), dirfd, path.trim())?; let (name, parent) = rsplit_path(&path); if let Some(parent) = parent { - current_inode = current_inode.lookup(parent)?; + current_inode = + current_inode.lookup_follow_symlink(parent, VFS_MAX_FOLLOW_SYMLINK_TIMES)?; } // debug!("mkdir at {:?}", current_inode.metadata()?.inode_id); return current_inode.mkdir(name, ModeType::from_bits_truncate(mode.bits())); diff --git a/kernel/src/filesystem/vfs/syscall.rs b/kernel/src/filesystem/vfs/syscall.rs index 53e674f47..545d05f0f 100644 --- a/kernel/src/filesystem/vfs/syscall.rs +++ b/kernel/src/filesystem/vfs/syscall.rs @@ -807,6 +807,14 @@ impl Syscall { return Ok(0); } + pub fn mkdir_at(dirfd: i32, path: *const u8, mode: usize) -> Result { + let path = check_and_clone_cstr(path, Some(MAX_PATHLEN))? + .into_string() + .map_err(|_| SystemError::EINVAL)?; + do_mkdir_at(dirfd, &path, FileMode::from_bits_truncate(mode as u32))?; + return Ok(0); + } + /// **创建硬连接的系统调用** /// /// ## 参数 diff --git a/kernel/src/syscall/mod.rs b/kernel/src/syscall/mod.rs index 7addb0c47..3502f6ff6 100644 --- a/kernel/src/syscall/mod.rs +++ b/kernel/src/syscall/mod.rs @@ -295,6 +295,13 @@ impl Syscall { Self::mkdir(path, mode) } + SYS_MKDIRAT => { + let dirfd = args[0] as i32; + let path = args[1] as *const u8; + let mode = args[2]; + Self::mkdir_at(dirfd, path, mode) + } + SYS_NANOSLEEP => { let req = args[0] as *const PosixTimeSpec; let rem = args[1] as *mut PosixTimeSpec; From cae618225763a6f7ddd7ee422d44414cd2d07561 Mon Sep 17 00:00:00 2001 From: LoGin Date: Fri, 18 Oct 2024 17:47:28 +0800 Subject: [PATCH 02/30] =?UTF-8?q?fix:=20=E4=B8=B4=E6=97=B6=E4=BF=AE?= =?UTF-8?q?=E5=A4=8D=E7=BA=BF=E7=A8=8B=E9=80=80=E5=87=BA=E6=97=B6=E5=A4=84?= =?UTF-8?q?=E7=90=86tty=E5=89=8D=E5=8F=B0=E8=BF=9B=E7=A8=8B=E7=BB=84?= =?UTF-8?q?=E4=BA=A7=E7=94=9Fpanic=E7=9A=84bug=20(#985)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kernel/src/process/fork.rs | 2 ++ kernel/src/process/mod.rs | 9 +++------ 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/kernel/src/process/fork.rs b/kernel/src/process/fork.rs index f750e4c27..547325c6c 100644 --- a/kernel/src/process/fork.rs +++ b/kernel/src/process/fork.rs @@ -166,6 +166,8 @@ impl ProcessManager { let pcb = ProcessControlBlock::new(name, new_kstack); + // TODO: 注意！这里设置tty的操作不符合Linux的行为！（毕竟创建进程不一定要fork，也可以用clone来创建） + // 正确做法应该是在实现进程组之后去管理前台进程组。 pcb.sig_info_mut() .set_tty(current_pcb.sig_info_irqsave().tty()); diff --git a/kernel/src/process/mod.rs b/kernel/src/process/mod.rs index fa35a988d..d5841e93b 100644 --- a/kernel/src/process/mod.rs +++ b/kernel/src/process/mod.rs @@ -425,12 +425,9 @@ impl ProcessManager { // TODO 由于未实现进程组，tty记录的前台进程组等于当前进程，故退出前要置空 // 后续相关逻辑需要在SYS_EXIT_GROUP系统调用中实现 - pcb.sig_info_irqsave() - .tty() - .unwrap() - .core() - .contorl_info_irqsave() - .pgid = None; + if let Some(tty) = pcb.sig_info_irqsave().tty() { + tty.core().contorl_info_irqsave().pgid = None; + } pcb.sig_info_mut().set_tty(None); drop(pcb); From 01c18c64b14b4ebabd98fa92c587c26874275eb1 Mon Sep 17 00:00:00 2001 From: sparkzky <146502758+sparkzky@users.noreply.github.com> Date: Sun, 20 Oct 2024 20:56:11 +0800 Subject: [PATCH 03/30] =?UTF-8?q?feat:=E6=B7=BB=E5=8A=A0symlink=E7=B3=BB?= =?UTF-8?q?=E7=BB=9F=E8=B0=83=E7=94=A8=20(#984)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 添加symlink系统调用 * 修改FATInode的dname的获取逻辑 * 修改fat对Dname的处理，分离dname和inode缓存的key --------- Co-authored-by: sparkzky Co-authored-by: longjin --- kernel/src/filesystem/fat/fs.rs | 56 ++++++++--------- kernel/src/filesystem/fat/utils.rs | 16 ++++- kernel/src/filesystem/vfs/core.rs | 50 ++++++++++++++- kernel/src/filesystem/vfs/mount.rs | 30 +++++++-- kernel/src/filesystem/vfs/syscall.rs | 13 ++++ kernel/src/syscall/mod.rs | 14 +++++ user/apps/test-mount/Cargo.toml | 3 +- user/apps/test-mount/src/main.rs | 5 +- user/apps/test-symlink/.gitignore | 3 + user/apps/test-symlink/Cargo.toml | 13 ++++ user/apps/test-symlink/Makefile | 56 +++++++++++++++++ user/apps/test-symlink/README.md | 1 + user/apps/test-symlink/src/main.rs | 78 ++++++++++++++++++++++++ user/dadk/config/test_symlink_0_1_0.dadk | 29 +++++++++ 14 files changed, 328 insertions(+), 39 deletions(-) create mode 100644 user/apps/test-symlink/.gitignore create mode 100644 user/apps/test-symlink/Cargo.toml create mode 100644 user/apps/test-symlink/Makefile create mode 100644 user/apps/test-symlink/README.md create mode 100644 user/apps/test-symlink/src/main.rs create mode 100644 user/dadk/config/test_symlink_0_1_0.dadk diff --git a/kernel/src/filesystem/fat/fs.rs b/kernel/src/filesystem/fat/fs.rs index 49b68222b..8e20210b5 100644 --- a/kernel/src/filesystem/fat/fs.rs +++ b/kernel/src/filesystem/fat/fs.rs @@ -2,11 +2,11 @@ use alloc::string::ToString; use core::cmp::Ordering; use core::intrinsics::unlikely; use core::{any::Any, fmt::Debug}; +use hashbrown::HashMap; use log::error; use system_error::SystemError; use alloc::{ - collections::BTreeMap, string::String, sync::{Arc, Weak}, vec::Vec, @@ -36,6 +36,7 @@ use crate::{ }; use super::entry::FATFile; +use super::utils::{to_search_name, to_search_name_string}; use super::{ bpb::{BiosParameterBlock, FATType}, entry::{FATDir, FATDirEntry, FATDirIter, FATEntry}, @@ -106,9 +107,9 @@ pub struct FATInode { parent: Weak, /// 指向自身的弱引用 self_ref: Weak, - /// 子Inode的B树. 该数据结构用作缓存区。其中，它的key表示inode的名称。 + /// 子Inode的map. 该数据结构用作缓存区。其中，它的key表示inode的名称。 /// 请注意，由于FAT的查询过程对大小写不敏感，因此我们选择让key全部是大写的，方便统一操作。 - children: BTreeMap>, + children: HashMap>, /// 当前inode的元数据 metadata: Metadata, /// 指向inode所在的文件系统对象的指针 @@ -148,24 +149,25 @@ impl FATInode { fn find(&mut self, name: &str) -> Result, SystemError> { match &self.inode_type { FATDirEntry::Dir(d) => { - let dname = DName::from(name.to_uppercase()); + let search_name = to_search_name(name); // 尝试在缓存区查找 - if let Some(entry) = self.children.get(&dname) { + if let Some(entry) = self.children.get(&search_name) { return Ok(entry.clone()); } // 在缓存区找不到 // 在磁盘查找 let fat_entry: FATDirEntry = d.find_entry(name, None, None, self.fs.upgrade().unwrap())?; + let dname = DName::from(name); // 创建新的inode let entry_inode: Arc = LockedFATInode::new( - dname.clone(), + dname, self.fs.upgrade().unwrap(), self.self_ref.clone(), fat_entry, ); // 加入缓存区, 由于FAT文件系统的大小写不敏感问题，因此存入缓存区的key应当是全大写的 - self.children.insert(dname, entry_inode.clone()); + self.children.insert(search_name, entry_inode.clone()); return Ok(entry_inode); } FATDirEntry::UnInit => { @@ -197,7 +199,7 @@ impl LockedFATInode { let inode: Arc = Arc::new(LockedFATInode(SpinLock::new(FATInode { parent, self_ref: Weak::default(), - children: BTreeMap::new(), + children: HashMap::new(), fs: Arc::downgrade(&fs), inode_type, metadata: Metadata { @@ -348,7 +350,7 @@ impl FATFileSystem { let root_inode: Arc = Arc::new(LockedFATInode(SpinLock::new(FATInode { parent: Weak::default(), self_ref: Weak::default(), - children: BTreeMap::new(), + children: HashMap::new(), fs: Weak::default(), inode_type: FATDirEntry::UnInit, metadata: Metadata { @@ -1559,14 +1561,15 @@ impl IndexNode for LockedFATInode { for ent in dir_iter { ret.push(ent.name()); - // ====== 生成inode缓存，存入B树 - let name = DName::from(ent.name().to_uppercase()); + // ====== 生成inode缓存 + let search_name = to_search_name_string(ent.name()); // debug!("name={name}"); - if !guard.children.contains_key(&name) - && name.as_ref() != "." - && name.as_ref() != ".." + if !guard.children.contains_key(&search_name) + && search_name != "." + && search_name != ".." { + let name = DName::from(ent.name()); // 创建新的inode let entry_inode: Arc = LockedFATInode::new( name.clone(), @@ -1575,7 +1578,7 @@ impl IndexNode for LockedFATInode { ent, ); // 加入缓存区, 由于FAT文件系统的大小写不敏感问题，因此存入缓存区的key应当是全大写的 - guard.children.insert(name, entry_inode.clone()); + guard.children.insert(search_name, entry_inode.clone()); } } return Ok(ret); @@ -1611,7 +1614,7 @@ impl IndexNode for LockedFATInode { // 对目标inode上锁，以防更改 let target_guard: SpinLockGuard = target.0.lock(); // 先从缓存删除 - let nod = guard.children.remove(&DName::from(name.to_uppercase())); + let nod = guard.children.remove(&to_search_name(name)); // 若删除缓存中为管道的文件，则不需要再到磁盘删除 if nod.is_some() { @@ -1646,7 +1649,7 @@ impl IndexNode for LockedFATInode { // 对目标inode上锁，以防更改 let target_guard: SpinLockGuard = target.0.lock(); // 先从缓存删除 - guard.children.remove(&DName::from(name.to_uppercase())); + guard.children.remove(&to_search_name(name)); let dir = match &guard.inode_type { FATDirEntry::File(_) | FATDirEntry::VolId(_) => { @@ -1669,9 +1672,7 @@ impl IndexNode for LockedFATInode { Err(r) => { if r == SystemError::ENOTEMPTY { // 如果要删除的是目录，且不为空，则删除动作未发生，重新加入缓存 - guard - .children - .insert(DName::from(name.to_uppercase()), target.clone()); + guard.children.insert(to_search_name(name), target.clone()); drop(target_guard); } return Err(r); @@ -1695,7 +1696,6 @@ impl IndexNode for LockedFATInode { let old_inode_guard: SpinLockGuard = old_inode.0.lock(); let fs = old_inode_guard.fs.upgrade().unwrap(); // 从缓存删除 - let _nod = guard.children.remove(&DName::from(old_name.to_uppercase())); let old_dir = match &guard.inode_type { FATDirEntry::File(_) | FATDirEntry::VolId(_) => { return Err(SystemError::ENOTDIR); @@ -1710,6 +1710,7 @@ impl IndexNode for LockedFATInode { // old_dir.check_existence(old_name, Some(false), guard.fs.upgrade().unwrap())?; old_dir.rename(fs, old_name, new_name)?; + let _nod = guard.children.remove(&to_search_name(old_name)); } else { let mut old_guard = self.0.lock(); let other: &LockedFATInode = target @@ -1721,10 +1722,7 @@ impl IndexNode for LockedFATInode { // 对目标inode上锁，以防更改 let old_inode_guard: SpinLockGuard = old_inode.0.lock(); let fs = old_inode_guard.fs.upgrade().unwrap(); - // 从缓存删除 - let _nod = old_guard - .children - .remove(&DName::from(old_name.to_uppercase())); + let old_dir = match &old_guard.inode_type { FATDirEntry::File(_) | FATDirEntry::VolId(_) => { return Err(SystemError::ENOTDIR); @@ -1748,6 +1746,8 @@ impl IndexNode for LockedFATInode { // 检查文件是否存在 old_dir.check_existence(old_name, Some(false), old_guard.fs.upgrade().unwrap())?; old_dir.rename_across(fs, new_dir, old_name, new_name)?; + // 从缓存删除 + let _nod = old_guard.children.remove(&to_search_name(old_name)); } return Ok(()); @@ -1806,9 +1806,9 @@ impl IndexNode for LockedFATInode { return self.create(filename, FileType::File, mode); } - let filename = DName::from(filename.to_uppercase()); + let dname = DName::from(filename); let nod = LockedFATInode::new( - filename.clone(), + dname, inode.fs.upgrade().unwrap(), inode.self_ref.clone(), FATDirEntry::File(FATFile::default()), @@ -1830,7 +1830,7 @@ impl IndexNode for LockedFATInode { return Err(SystemError::EINVAL); } - inode.children.insert(filename, nod.clone()); + inode.children.insert(to_search_name(filename), nod.clone()); Ok(nod) } diff --git a/kernel/src/filesystem/fat/utils.rs b/kernel/src/filesystem/fat/utils.rs index 97f5a8236..329cb84a9 100644 --- a/kernel/src/filesystem/fat/utils.rs +++ b/kernel/src/filesystem/fat/utils.rs @@ -1,3 +1,4 @@ +use alloc::string::String; use core::char::REPLACEMENT_CHARACTER; /// FAT文件系统保留开头的2个簇 @@ -5,7 +6,7 @@ pub const RESERVED_CLUSTERS: u32 = 2; /// @brief 将u8转为ascii字符。 /// 当转码成功时，返回对应的ascii字符，否则返回Unicode占位符 -pub fn decode_u8_ascii(value: u8) -> char { +pub(super) fn decode_u8_ascii(value: u8) -> char { if value <= 0x7f { return value as char; } else { @@ -13,3 +14,16 @@ pub fn decode_u8_ascii(value: u8) -> char { return REPLACEMENT_CHARACTER; } } + +/// 把名称转为inode缓存里面的key +#[inline(always)] +pub(super) fn to_search_name(name: &str) -> String { + name.to_ascii_uppercase() +} + +/// 把名称转为inode缓存里面的key(输入为string，原地替换) +#[inline(always)] +pub(super) fn to_search_name_string(mut name: String) -> String { + name.make_ascii_uppercase(); + name +} diff --git a/kernel/src/filesystem/vfs/core.rs b/kernel/src/filesystem/vfs/core.rs index e519417bd..85ea2c4f2 100644 --- a/kernel/src/filesystem/vfs/core.rs +++ b/kernel/src/filesystem/vfs/core.rs @@ -12,9 +12,13 @@ use crate::{ procfs::procfs_init, ramfs::RamFS, sysfs::sysfs_init, - vfs::{mount::MountFS, syscall::ModeType, AtomicInodeId, FileSystem, FileType}, + vfs::{ + mount::MountFS, syscall::ModeType, AtomicInodeId, FileSystem, FileType, MAX_PATHLEN, + }, }, + libs::spinlock::SpinLock, process::ProcessManager, + syscall::user_access::check_and_clone_cstr, }; use super::{ @@ -23,7 +27,7 @@ use super::{ mount::{init_mountlist, MOUNT_LIST}, syscall::UmountFlag, utils::{rsplit_path, user_path_at}, - IndexNode, InodeId, VFS_MAX_FOLLOW_SYMLINK_TIMES, + FilePrivateData, IndexNode, InodeId, VFS_MAX_FOLLOW_SYMLINK_TIMES, }; /// 当没有指定根文件系统时，尝试的根文件系统列表 @@ -248,6 +252,48 @@ pub fn do_unlink_at(dirfd: i32, path: &str) -> Result { return Ok(0); } +pub fn do_symlinkat(from: *const u8, newdfd: i32, to: *const u8) -> Result { + let oldname = check_and_clone_cstr(from, Some(MAX_PATHLEN))? + .into_string() + .map_err(|_| SystemError::EINVAL)?; + let newname = check_and_clone_cstr(to, Some(MAX_PATHLEN))? + .into_string() + .map_err(|_| SystemError::EINVAL)?; + let from = oldname.as_str().trim(); + let to = newname.as_str().trim(); + + // TODO: 添加权限检查，确保进程拥有目标路径的权限 + + let pcb = ProcessManager::current_pcb(); + let (old_begin_inode, old_remain_path) = user_path_at(&pcb, AtFlags::AT_FDCWD.bits(), from)?; + // info!("old_begin_inode={:?}", old_begin_inode.metadata()); + let _ = + old_begin_inode.lookup_follow_symlink(&old_remain_path, VFS_MAX_FOLLOW_SYMLINK_TIMES)?; + + // 得到新创建节点的父节点 + let (new_begin_inode, new_remain_path) = user_path_at(&pcb, newdfd, to)?; + let (new_name, new_parent_path) = rsplit_path(&new_remain_path); + let new_parent = new_begin_inode + .lookup_follow_symlink(new_parent_path.unwrap_or("/"), VFS_MAX_FOLLOW_SYMLINK_TIMES)?; + // info!("new_parent={:?}", new_parent.metadata()); + + if new_parent.metadata()?.file_type != FileType::Dir { + return Err(SystemError::ENOTDIR); + } + + let new_inode = new_parent.create_with_data( + new_name, + FileType::SymLink, + ModeType::from_bits_truncate(0o777), + 0, + )?; + + let buf = old_remain_path.as_bytes(); + let len = buf.len(); + new_inode.write_at(0, len, buf, SpinLock::new(FilePrivateData::Unused).lock())?; + return Ok(0); +} + /// # do_mount - 挂载文件系统 /// /// 将给定的文件系统挂载到指定的挂载点。 diff --git a/kernel/src/filesystem/vfs/mount.rs b/kernel/src/filesystem/vfs/mount.rs index c4ab90791..12f122b8c 100644 --- a/kernel/src/filesystem/vfs/mount.rs +++ b/kernel/src/filesystem/vfs/mount.rs @@ -8,6 +8,7 @@ use alloc::{ collections::BTreeMap, string::{String, ToString}, sync::{Arc, Weak}, + vec::Vec, }; use system_error::SystemError; @@ -215,12 +216,29 @@ impl MountFSInode { .ok_or(SystemError::ENOENT); } - fn do_absolute_path(&self, len: usize) -> Result { - if self.metadata()?.inode_id == ROOT_INODE().metadata()?.inode_id { - return Ok(String::with_capacity(len)); + fn do_absolute_path(&self) -> Result { + let mut path_parts = Vec::new(); + let mut current = self.self_ref.upgrade().unwrap(); + + while current.metadata()?.inode_id != ROOT_INODE().metadata()?.inode_id { + let name = current.dname()?; + path_parts.push(name.0); + current = current.do_parent()?; } - let name = self.dname()?; - return Ok(self.do_parent()?.do_absolute_path(len + name.0.len() + 1)? + "/" + &name.0); + + // 由于我们从叶子节点向上遍历到根节点，所以需要反转路径部分 + path_parts.reverse(); + + // 构建最终的绝对路径字符串 + let mut absolute_path = String::with_capacity( + path_parts.iter().map(|s| s.len()).sum::() + path_parts.len(), + ); + for part in path_parts { + absolute_path.push('/'); + absolute_path.push_str(&part); + } + + Ok(absolute_path) } } @@ -469,7 +487,7 @@ impl IndexNode for MountFSInode { } fn absolute_path(&self) -> Result { - self.do_absolute_path(0) + self.do_absolute_path() } #[inline] diff --git a/kernel/src/filesystem/vfs/syscall.rs b/kernel/src/filesystem/vfs/syscall.rs index 545d05f0f..5d256bf03 100644 --- a/kernel/src/filesystem/vfs/syscall.rs +++ b/kernel/src/filesystem/vfs/syscall.rs @@ -20,6 +20,7 @@ use crate::{ time::{syscall::PosixTimeval, PosixTimeSpec}, }; +use super::core::do_symlinkat; use super::{ core::{do_mkdir_at, do_remove_dir, do_unlink_at}, fcntl::{AtFlags, FcntlCommand, FD_CLOEXEC}, @@ -980,6 +981,18 @@ impl Syscall { return do_unlink_at(AtFlags::AT_FDCWD.bits(), &path).map(|v| v as usize); } + pub fn symlink(oldname: *const u8, newname: *const u8) -> Result { + return do_symlinkat(oldname, AtFlags::AT_FDCWD.bits(), newname); + } + + pub fn symlinkat( + oldname: *const u8, + newdfd: i32, + newname: *const u8, + ) -> Result { + return do_symlinkat(oldname, newdfd, newname); + } + /// # 修改文件名 /// /// diff --git a/kernel/src/syscall/mod.rs b/kernel/src/syscall/mod.rs index 3502f6ff6..c61c2db57 100644 --- a/kernel/src/syscall/mod.rs +++ b/kernel/src/syscall/mod.rs @@ -346,6 +346,20 @@ impl Syscall { Self::unlinkat(dirfd, path, flags) } + #[cfg(target_arch = "x86_64")] + SYS_SYMLINK => { + let oldname = args[0] as *const u8; + let newname = args[1] as *const u8; + Self::symlink(oldname, newname) + } + + SYS_SYMLINKAT => { + let oldname = args[0] as *const u8; + let newdfd = args[1] as i32; + let newname = args[2] as *const u8; + Self::symlinkat(oldname, newdfd, newname) + } + #[cfg(target_arch = "x86_64")] SYS_RMDIR => { let path = args[0] as *const u8; diff --git a/user/apps/test-mount/Cargo.toml b/user/apps/test-mount/Cargo.toml index a379b0601..912080a3e 100644 --- a/user/apps/test-mount/Cargo.toml +++ b/user/apps/test-mount/Cargo.toml @@ -8,4 +8,5 @@ authors = [ "xiaolin2004 <1553367438@qq.com>" ] # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -libc="0.2" \ No newline at end of file +errno = "0.3.9" +libc="0.2" diff --git a/user/apps/test-mount/src/main.rs b/user/apps/test-mount/src/main.rs index 271310d04..10279cb6c 100644 --- a/user/apps/test-mount/src/main.rs +++ b/user/apps/test-mount/src/main.rs @@ -1,8 +1,10 @@ use core::ffi::{c_char, c_void}; +use errno::errno; use libc::{mount, MS_BIND}; use std::fs; use std::path::Path; use std::time; + fn main() { let path = Path::new("mnt/tmp"); let dir = fs::create_dir_all(path); @@ -26,7 +28,8 @@ fn main() { if result == 0 { println!("Mount successful"); } else { - println!("Mount failed"); + let err = errno(); + println!("Mount failed with error code: {}", err.0); } let dur = clock.elapsed(); println!("mount costing time: {} ns", dur.as_nanos()); diff --git a/user/apps/test-symlink/.gitignore b/user/apps/test-symlink/.gitignore new file mode 100644 index 000000000..7ca01a554 --- /dev/null +++ b/user/apps/test-symlink/.gitignore @@ -0,0 +1,3 @@ +/target +Cargo.lock +/install/ diff --git a/user/apps/test-symlink/Cargo.toml b/user/apps/test-symlink/Cargo.toml new file mode 100644 index 000000000..bbad89e7c --- /dev/null +++ b/user/apps/test-symlink/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "test-symlink" +version = "0.1.0" +edition = "2021" +description = "测试symlink系统调用" +authors = [ "sparkzky " ] + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +errno = "0.3.9" +libc="0.2" +nix = "0.23" \ No newline at end of file diff --git a/user/apps/test-symlink/Makefile b/user/apps/test-symlink/Makefile new file mode 100644 index 000000000..7522ea16c --- /dev/null +++ b/user/apps/test-symlink/Makefile @@ -0,0 +1,56 @@ +TOOLCHAIN= +RUSTFLAGS= + +ifdef DADK_CURRENT_BUILD_DIR +# 如果是在dadk中编译，那么安装到dadk的安装目录中 + INSTALL_DIR = $(DADK_CURRENT_BUILD_DIR) +else +# 如果是在本地编译，那么安装到当前目录下的install目录中 + INSTALL_DIR = ./install +endif + +ifeq ($(ARCH), x86_64) + export RUST_TARGET=x86_64-unknown-linux-musl +else ifeq ($(ARCH), riscv64) + export RUST_TARGET=riscv64gc-unknown-linux-gnu +else +# 默认为x86_86，用于本地编译 + export RUST_TARGET=x86_64-unknown-linux-musl +endif + +run: + RUSTFLAGS=$(RUSTFLAGS) cargo $(TOOLCHAIN) run --target $(RUST_TARGET) + +build: + RUSTFLAGS=$(RUSTFLAGS) cargo $(TOOLCHAIN) build --target $(RUST_TARGET) + +clean: + RUSTFLAGS=$(RUSTFLAGS) cargo $(TOOLCHAIN) clean --target $(RUST_TARGET) + +test: + RUSTFLAGS=$(RUSTFLAGS) cargo $(TOOLCHAIN) test --target $(RUST_TARGET) + +doc: + RUSTFLAGS=$(RUSTFLAGS) cargo $(TOOLCHAIN) doc --target $(RUST_TARGET) + +fmt: + RUSTFLAGS=$(RUSTFLAGS) cargo $(TOOLCHAIN) fmt + +fmt-check: + RUSTFLAGS=$(RUSTFLAGS) cargo $(TOOLCHAIN) fmt --check + +run-release: + RUSTFLAGS=$(RUSTFLAGS) cargo $(TOOLCHAIN) run --target $(RUST_TARGET) --release + +build-release: + RUSTFLAGS=$(RUSTFLAGS) cargo $(TOOLCHAIN) build --target $(RUST_TARGET) --release + +clean-release: + RUSTFLAGS=$(RUSTFLAGS) cargo $(TOOLCHAIN) clean --target $(RUST_TARGET) --release + +test-release: + RUSTFLAGS=$(RUSTFLAGS) cargo $(TOOLCHAIN) test --target $(RUST_TARGET) --release + +.PHONY: install +install: + RUSTFLAGS=$(RUSTFLAGS) cargo $(TOOLCHAIN) install --target $(RUST_TARGET) --path . --no-track --root $(INSTALL_DIR) --force diff --git a/user/apps/test-symlink/README.md b/user/apps/test-symlink/README.md new file mode 100644 index 000000000..6fe2bdc65 --- /dev/null +++ b/user/apps/test-symlink/README.md @@ -0,0 +1 @@ +# 测试Symlink系统调用的程序 diff --git a/user/apps/test-symlink/src/main.rs b/user/apps/test-symlink/src/main.rs new file mode 100644 index 000000000..af070694f --- /dev/null +++ b/user/apps/test-symlink/src/main.rs @@ -0,0 +1,78 @@ +extern crate libc; +use core::ffi::{c_char, c_void}; +use libc::{mount, umount}; +use nix::errno::Errno; +use std::fs; +use std::os::unix::fs::symlink; +use std::path::Path; + +fn main() { + mount_test_ramfs(); + + let target = "/mnt/myramfs/target_file.txt"; + let symlink_path = "/mnt/myramfs/another/symlink_file.txt"; + let dir = "/mnt/myramfs/another"; + + fs::write(target, "This is the content of the target file.") + .expect("Failed to create target file"); + fs::create_dir(dir).expect("Failed to create target dir"); + + assert!(Path::new(target).exists(), "Target file was not created"); + assert!(Path::new(dir).exists(), "Target dir was not created"); + + symlink(target, symlink_path).expect("Failed to create symlink"); + + assert!(Path::new(symlink_path).exists(), "Symlink was not created"); + + let symlink_content = fs::read_link(symlink_path).expect("Failed to read symlink"); + assert_eq!( + symlink_content.display().to_string(), + target, + "Symlink points to the wrong target" + ); + + fs::remove_file(symlink_path).expect("Failed to remove symlink"); + fs::remove_file(target).expect("Failed to remove target file"); + fs::remove_dir(dir).expect("Failed to remove test_dir"); + + assert!(!Path::new(symlink_path).exists(), "Symlink was not deleted"); + assert!(!Path::new(target).exists(), "Target file was not deleted"); + assert!(!Path::new(dir).exists(), "Directory was not deleted"); + + umount_test_ramfs(); + + println!("All tests passed!"); +} + +fn mount_test_ramfs() { + let path = Path::new("mnt/myramfs"); + let dir = fs::create_dir_all(path); + assert!(dir.is_ok(), "mkdir /mnt/myramfs failed"); + + let source = b"\0".as_ptr() as *const c_char; + let target = b"/mnt/myramfs\0".as_ptr() as *const c_char; + let fstype = b"ramfs\0".as_ptr() as *const c_char; + // let flags = MS_BIND; + let flags = 0; + let data = std::ptr::null() as *const c_void; + let result = unsafe { mount(source, target, fstype, flags, data) }; + + assert_eq!( + result, + 0, + "Mount myramfs failed, errno: {}", + Errno::last().desc() + ); + println!("Mount myramfs success!"); +} + +fn umount_test_ramfs() { + let path = b"/mnt/myramfs\0".as_ptr() as *const c_char; + let result = unsafe { umount(path) }; + if result != 0 { + let err = Errno::last(); + println!("Errno: {}", err); + println!("Infomation: {}", err.desc()); + } + assert_eq!(result, 0, "Umount myramfs failed"); +} diff --git a/user/dadk/config/test_symlink_0_1_0.dadk b/user/dadk/config/test_symlink_0_1_0.dadk new file mode 100644 index 000000000..ea5b0dff2 --- /dev/null +++ b/user/dadk/config/test_symlink_0_1_0.dadk @@ -0,0 +1,29 @@ +{ + "name": "test-symlink", + "version": "0.1.0", + "description": "测试symlink系统调用", + "rust_target": "x86_64-unknown-dragonos", + "task_type": { + "BuildFromSource": { + "Local": { + "path": "apps/test-symlink" + } + } + }, + "depends": [], + "build": { + "build_command": "make install" + }, + "install": { + "in_dragonos_path": "/" + }, + "clean": { + "clean_command": "make clean" + }, + "envs": [], + "build_once": false, + "install_once": false, + "target_arch": [ + "x86_64" + ] +} \ No newline at end of file From abc421a67f4aaf473e8028a207222fc3986493d8 Mon Sep 17 00:00:00 2001 From: LoGin Date: Mon, 21 Oct 2024 00:16:14 +0800 Subject: [PATCH 04/30] =?UTF-8?q?=E4=BF=AE=E6=AD=A3=E5=BC=80=E5=8F=91?= =?UTF-8?q?=E8=80=85=E5=8F=82=E4=B8=8E=E6=8C=87=E5=8D=97=E7=9A=84=E9=93=BE?= =?UTF-8?q?=E6=8E=A5=20(#1002)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/Makefile | 3 + docs/community/code_contribution/index.rst | 3 +- .../the-development-process.md | 155 ------------------ docs/community/contact/index.rst | 4 +- triagebot.toml | 2 +- 5 files changed, 7 insertions(+), 160 deletions(-) delete mode 100644 docs/community/code_contribution/the-development-process.md diff --git a/docs/Makefile b/docs/Makefile index 92615f16a..f08b586c1 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -36,3 +36,6 @@ help: .PHONY: html-multiversion: rm -rf ./$(BUILDDIR) && CURRENT_GIT_COMMIT_DIRTY=0 sphinx-multiversion $(SPHINXOPTS) "$(SOURCEDIR)" ./$(BUILDDIR)/html && cp -rf ./$(BUILDDIR)/html/master/* ./$(BUILDDIR)/html/ + +http_server: + python3 -m http.server --directory $(BUILDDIR)/html diff --git a/docs/community/code_contribution/index.rst b/docs/community/code_contribution/index.rst index e849ca113..4a6aab77c 100644 --- a/docs/community/code_contribution/index.rst +++ b/docs/community/code_contribution/index.rst @@ -9,7 +9,8 @@ .. toctree:: :maxdepth: 1 - the-development-process + how-to-contribute + c-coding-style rust-coding-style conventional-commit \ No newline at end of file diff --git a/docs/community/code_contribution/the-development-process.md b/docs/community/code_contribution/the-development-process.md deleted file mode 100644 index c1a2f6a2c..000000000 --- a/docs/community/code_contribution/the-development-process.md +++ /dev/null @@ -1,155 +0,0 @@ -# 开发流程介绍 - -  作为一名想要参与开发的新人，您可能迫切想要了解如何参与开发，仔细阅读这篇文章将能帮助您了解整个开发流程，以及一些注意事项。 - -  注：本文参考了Linux文档中的一些思想、内容，非常感谢Linux社区的工作者们的经验！ - -## 1.概要 - -对于新人而言，参与开发的过程主要包括以下几步： - -- **运行DragonOS**：按照文档：{ref}`构建DragonOS <_build_dragonos>`中的教程，编译DragonOS，并成功运行。在运行过程中，如果有任何的疑问，欢迎您在交流群或者BBS上进行反馈！ -- **联系Maintainer**：您可以通过邮箱或者QQ`184829088`与龙进取得联系，或者是对应的模块的开发者进行联系（目前您可以通过发行日志上的邮箱与他们建立联系，在将来我们将编写一份“Maintainers List”以便于您能快速找到要联系的人）。 - 为了节省您的时间，请简要说明： - - 如何称呼您 - - 您目前掌握的技术能力 - - 您希望为DragonOS贡献什么功能，或者进行某个方面的开发，亦或者是想要按照社区的需要来参与新功能开发及bug的修复。 - - 如果您是来自高校/科研单位/企业/组织的代表，希望与社区开展合作研究、开发。那么，除使用QQ交流之外，还请麻烦您通过您的教师邮箱/学生邮箱/企业邮箱向发送一封相关内容的邮件！这么做的目的是为了确认您是来自您的单位，而不是网络上其他人员冒充您的身份。 -- **加入工作群**：在进一步了解，确认您愿意参与开发后，我们将邀请您加入工作群。 -- **开始开发**：正式开始代码开发工作。 - -:::{note} -一些小功能的改进以及Bug修复并不一定需要提前与社区建立正式的联系，对于这些patch，您可以直接开发，然后在Github上进行Pull Request. 这也是可以的。 - -但是，如果您愿意的话，与Maintainer联系会是一个更好的选择。 -::: - -## 2.开发过程是如何运作的？ - -  如今的DragonOS由于正处于开发的早期阶段，开发者数量不超过50人，因此，现在DragonOS的开发过程是通过比较松散的方式组织起来的。 - -### 2.1.版本发布周期 - -  自从2022年11月6日，DragonOS发布第一个版本以来，版本发布就被定义为15~21天发布一个更新版本。由于开发人员数量仍然较少，因此，目前这个时间是21天。我们将版本号定义为`x.y.z`的格式，每21天发布一个`z`版本. 在积累了2～3个月后，当DragonOS引入了足够的新功能，则发布一个`y`版本。请注意，我们仍未定义`x`版本的发行周期。当前，`x`版本号仍然是`0`。 - -  创建没有BUG的、具有尽可能少BUG的代码，是每个开发者的目标之一。我们希望在每个`y`版本发布时，能够修复已知的问题。然而，由于在操作系统中，影响问题的变量太多了，以至于我们只能尽全力去减少BUG，我们无法保证`y`版本不存在bug. - -### 2.2.每个补丁的生命周期 - -  当您向DragonOS的仓库发起一次PR，那么这次PR就是一个补丁。我们会对您的补丁进行Review，以确保每个补丁都实现了一个希望在主线中进行的更改。并且，Maintainer或者感兴趣的小伙伴会对您的补丁提出修改意见。当时机合适时，您的代码将被合入主线。 - -  如果您的补丁的规模比较小，那么，它将会比较快的被合入主线。如果补丁的规模较大，或者存在一些争议，那么我们需要对其进行进一步的讨论及修改、审查，直到它符合要求。 - -  每个Patch都会经历这么一个过程（这只是一个概述，详细的描述请看后文）： - -- **设计**：在这个阶段，我们将明确，这个补丁将要实现什么功能，或者是解决什么问题，以及我们将要采用什么样的方式去完成它。通常来说，这项工作是开发者自己完成的。但是，**我们建议您，在设计了这个补丁之后，能够把您的设计方案公开，和大家讨论这项工作。** 闭门造车容易出错，在与大家沟通的过程中，则能及早的发现设计上的错误，从而节省很多的时间。 -- **代码编写**：经过了设计阶段，您已经能够明白自己要实现的到底是一个什么样的东西。您在这个阶段进行代码编写、调试。 -- **代码审查**：当完成代码编写后，您可以通过Github发起一个PR，然后您的补丁进入了代码审查阶段。在这一阶段，开发者们，或者是Maintainer会和您进行沟通交流，对您的代码进行评论，以及对发现的问题提出修改建议。 -- **合并主线**：如果一切顺利，那么代码将会被合并入主线。若该补丁在合并主线后，被发现具有较大的问题，那么它可能会被回退。重新进入前面的阶段，进行修改。 -- **长期维护**：虽然说，代码被合并之后，原来的开发人员可能会在很久一段时间后，不再理会这些代码，但是这种行为可能会给其他开发者们留下不好的印象。其实，当补丁被合并入主线后，其他开发人员会尝试继续维护这些代码，这样能够很大程度的减轻您的维护负担。但是，如果想要这些代码能够长期的被保留下来，持续的发光发热，那么离不开原始开发人员的支持（不然的话，后来的人可能难以理解、维护这些代码），这是一件很有意义的事情。 - -  对于没有参与过开源项目的同学来说，他们可能会想当然的，简单的把上述流程简化成**合并主线**这一个步骤，这样是不可取的。因为这样会导致很多问题，包括但不限于“写了代码但是效果很差”、“写的东西由于无法满足项目的需求，而不能被合并”。 - -### 2.3.开发工具 - -  从上面的描述可以看出，为了让开发人员们能高效地进行协作，那么必须使用版本控制工具来管理这个过程。目前，DragonOS使用Git来进行源代码管理。它是一个非常优秀的工具，这是不必多说的。对于每个开发者而言，Git的使用是一项必备的技能；哪怕您只是想学习DragonOS的源代码，您也需要git来获取、同步最新的代码。虽然Git的使用，对于新手来说，有些困难，但是经过一些时间的学习后，还是可以掌握的。 - -  git的官方网站为[https://git-scm.com/](https://git-scm.com/) - -### 2.4.沟通交流 - -  DragonOS的主要开发工作是通过飞书以及bbs进行的。对于正准备参与的开发者而言，您可以加入我们的交流讨论QQ群，具体的群号可以在 {ref}`与社区建立联系 ` 一文中找到。 - -  **何时使用即时通讯软件？** 我们在飞书上创建了工作群，为提高即时沟通的效率，这个群仅供那些真正有意愿、且正在进行或准备进行（能够保证会进行）代码开发的开发者们加入。 - -  **何时使用BBS？** 对于一些正式的、需要大家广泛参与，或者是能够帮助尚未参与开发的同学了解当前的开发进度的主题，请您在[https://bbs.DragonOS.org](https://bbs.DragonOS.org)上，使用类似于写信件一样的，正式的语言，完整地描述清楚您想表达的内容。这样有助于更多的人快速明白您要表达的是什么，也能提高整体的沟通效率。并且，bbs能够长期保存以往的帖子，这样后来者能更方便的了解“当初开发的时候，人们究竟是怎么想的”。 - -  **关于交流讨论会** 除由于法定节假日放假，或特殊情况以外，我们每周末都会召开线上交流讨论会，同步介绍每周的进展。社区成员可以在这里分享自己的方案设计或是一些操作系统相关的知识（分享的话，需要提前跟会议主持人提出，以便妥善安排）。 - -  **如何提问？** 下面这些建议能够帮助您与他人开展高效率的对话： - -- **对于具有主题性的问题，在bbs上发帖进行讨论。** 这样能够让讨论更具有目标性。当谈论到新的主题的时候，请开一个新的帖子，并在原来的帖子中，添加对特定的子问题所在的主题的链接。 -- **请礼貌的交流。** 文明的语言能够减少不必要的冲突。技术意见上的冲突是思维的碰撞，但是如果涉及到了不文明的语言，或者在非技术层面，对他人进行攻击，这将破坏和谐讨论的氛围，这是我们反对的。如果有人试图激怒你，请忽略他的消息，别理他就好了。 -- **在提问之前，请确保您已经搜索了bbs以及互联网上的解决方案，并描述清楚您的问题的上下文情景、您的思考以及网络上的解决方案。** 一些开发人员会对“明显没有进行认真思考”的问题，表现出不耐烦的态度（因为未经思考的问题会浪费他们大量的时间）。 -- **当别人向您提问时**，请您耐心听他人的问题。如果您认为对方的问题过于简单或是未经思考，还请您为对方指个路，告诉对方，在哪些地方，使用什么样的方式搜索，能够得到对解决问题有帮助的资料。有时候，**新手需要的是一个指路人**，他会非常感谢您的！ - -### 2.5.如何入门开发？ - -  DragonOS原采用C语言进行开发，目前正在用Rust重构原有代码、开发新的模块，也就是说，除非您要进行对C语言代码的BUG修复，否则，其余的开发工作，我们都建议您通过Rust完成。因为，它能从语言层面解决那些让我们头疼的内存安全问题。从长期来看，能够提升开发效率以及软件质量。 - -  如何开发第一个补丁，是一个非常常见的问题。可以理解的是，个人开发者面对这样一个项目，常常会不知道从哪个地方开始入手。这是一件很正常的事情，因此我们建议您通过上文提到的方式，与社区建立联系，了解目前社区正在做什么，以及需要什么。 - -  对于一个新的参与者来说，我们建议您从这样一个步骤开始： - -```text -阅读文档，编译、运行DragonOS，并且尝试使用它目前已有的功能。 -``` - -  然后，您可以通过查看DragonOS的GitHub仓库的project面板，看看目前仍有哪些待解决的问题。可以肯定的是，永远不会缺少待解决的问题，您在解决这些问题的过程中，能够获得一些宝贵的经验。 - -## 3.早期设计 - -  对于软件开发而言，写代码永远不是第一步。在编写代码之前，进行一些必要的设计（提出架构、技术方案），是项目成功的基础工作。在新的补丁开发的早期，花费一些时间进行计划和沟通，往往能够在接下来的阶段节省更多的时间。 - -### 3.1.定义我们要解决的问题 - -  与大多数的工程项目一样，在DragonOS中进行开发，首先需要清晰的描述要解决的问题。只有精准的定义了问题，才能找到正确的解决方案。有时候，我们能很轻易的定义问题，比如“编写串口驱动程序，使得它能把屏幕上打印的字符，输出到串口”。 - -  但是，有时候，**我们容易将真正的问题与某个解决方案相混淆**，并且还没意识到这一点。 - -  在2022年10月时，我发现，在真机调试的时候，需要频繁的拔插硬盘（先连接到电脑，待数据拷贝完毕后，再连接到测试机）。我对这一过程非常的不满，因为很浪费时间。我的直觉想法是：“有没有一种设备，能够一头连接电脑，另一头连接测试机的SATA接口。从测试机的角度看来，这是一块硬盘；测试机对这块磁盘的操作，都转换为了对我的电脑上面的一个磁盘镜像文件的操作。”我的想法就是：“购买/定制一款设备，能够实现上面的这个功能，那就能解决频繁拔插硬盘的烦恼了！”然后我为了寻找这样的设备，询问了一些硬件公司，他们的开价都在2万元左右。 - -  我在上面的这个过程中，就犯了一个错误：将真正的问题与某个解决方案相混淆了。真正的问题是：“解决需要频繁的拔插硬盘”，但是，在我的思考的过程中，不知不觉间，将问题转换成了“如何实现一款具有硬盘模拟功能的设备”。后面这个问题，只是某个解决方案下，需要解决的问题，并不是我们要解决的根本问题。 - -  对于要解决的根本问题，事实上有更好的解决方案：“制作一个类似于开关一样的转换器，当数据从电脑拷贝到磁盘后，把开关拨向另一边，使得电路与测试机接通”。这个方案的成本估摸着就十几二十块钱。 - -  上面的这个故事，告诉我们的是，**在早期设计阶段，我们应当关注的是问题本身——而不是特定的解决方案**。 - -  我们需要关注系统的稳定性、长期的可维护性，解决方案必须考虑到这两点。由于系统比较复杂，因此，请您在开始编码之前，与社区的小伙伴讨论您的设计方案，以便您的方案能充分地，从全局的角度，考虑到系统的稳定性、可维护性。 - -  **因此，在开发的早期，我们应当对以下三个问题，拥有一个答案**： - -- 要解决的本质问题是什么？ -- 这个问题会影响哪些方面/哪些用户？提出的解决方案应当解决哪些用例、场景？ -- DragonOS目前在解决该问题的方面，具有哪些不足/问题？ - -  只有考虑清楚了上面三个问题，讨论的解决方案才是有意义的。这是一个架构设计的过程，需要进行仔细的思考。尽管我们目前提倡敏捷开发，但是前期的架构设计仍然是非常重要的。 - -### 3.2.早期讨论 - -  在实施开发之前，与社区的成员们进行讨论是非常有意义的。这能够通过多种方式节省您的时间，并减少许多不必要的麻烦： - -- DragonOS可能以您不知道、不理解的方式，已经解决了相关的问题。DragonOS里面的一些特性、功能细节不是很明显，他们不一定会被写进文档。也许这些细节只是在某个不起眼的注释里面提到了，因此您很难注意到这些。这种细节可能只有一些开发人员知道。因此，与社区沟通能够避免您进行重复的工作。 -- 您提出的解决方案中，可能会有一些东西，由于一些原因（比如方案中的一些设计会在将来造成问题、方案的架构设计具有明显缺陷），无法合入主线。 -- 其他的开发人员可能已经考虑过这个问题；他们可能有更好的解决方案，或者是更好的想法。并且，他们可能愿意帮助你一起完善你的解决方案。 - -  Linux文档中提到：闭门造车式的设计和开发，所产生的代码总会有问题，这些问题只有在发布到社区里面的时候才会暴露出来。因此，我们必须吸取前人之鉴，通过与社区开发人员进行早期讨论，从而避免大量的痛苦和额外的工作。 - - -### 3.3.在何时发布帖子？ - -  如果可能的话，在开发的早期阶段发布您的计划、设计，会是一个不错的选择。发帖的时候，您可以描述您正在解决的问题，以及已经制定的一些计划。包括但不限于：如何将设计付诸实施。您在社区内发布帖子，不仅能够帮助您获得一些有用的建议，也能帮助整个DragonOS社区提供有用的信息，使得社区沟通更高效。 - -  在这个阶段，可能您发布的帖子并不会引来很多评论，这并不一定意味着您做的不好，或者大家对您所做的工作不感兴趣。当然，也不能就此认为您的设计、想法不存在问题。可能只是因为大家比较忙，看了您的帖子之后，了解到了您的工作，但是大家并不一定有时间进行回复。（但是事实上您发布的信息对他人来说是有用的） - -  在这种情况下，请不要气馁，您最好的做法就是，继续您的工作，并且不时地在您的帖子下分享您的工作，这样能够让社区的成员们随时了解到您的最新进展。 - -### 3.4.获得您所在的组织的支持 - -  如果您对DragonOS的开发工作，是在您的公司内完成的。那么，很显然，在您把计划、代码发布到社区论坛之前，您必须取得您的经理或上级的许可。 - -  同时，请注意，根据我们的授权许可，基于DragonOS操作系统的内核、官方开源的用户库而开发的代码，或者为DragonOS操作系统本身而开发的代码，根据开源授权许可，必须同样以GPLv2协议进行授权发布。如果您所在的组织，违背了GPLv2协议中的要求，以除GPLv2以外的协议开放源代码，或者是进行“闭源使用”，那么DragonOS社区对您的公司/组织所授予的使用DragonOS源代码的授权，将会被自动撤销。这将会面临一系列的法律问题。因此，在这个问题上，公司的管理人员、法律人员如果能越早地就公司要在DragonOS中开发的软件项目达成一致，将能促进您的公司在该项目上的进展。 - -  如果您的公司的项目/或者是您研究的项目根据您所在组织的保密规定，不能将其进行过早的披露，那也没有问题。只要您的公司能够确保这部分代码，在其编译而成的二进制产品被发布之时，按照GPLv2协议进行开源，并向开源社区贡献这部分代码即可。 - -## 4.如何正确的编写代码 - -## 5.发起Pull Request - -## 6.后期跟进 - -## 7.另外的一些话题 - -## 8.更多信息 - -## 9.结语 \ No newline at end of file diff --git a/docs/community/contact/index.rst b/docs/community/contact/index.rst index f5078b989..b0baaa082 100644 --- a/docs/community/contact/index.rst +++ b/docs/community/contact/index.rst @@ -8,9 +8,7 @@ 社区公共邮箱：contact@DragonOS.org -DragonOS社区负责人: 龙进 - -工作邮箱: longjin@DragonOS.org +社区管理人员信息：https://community.dragonos.org/governance/staff-info.html 开发交流QQ群： 115763565 diff --git a/triagebot.toml b/triagebot.toml index 887cec0c3..d723edb01 100644 --- a/triagebot.toml +++ b/triagebot.toml @@ -110,7 +110,7 @@ cc = ["@GnoCiYeH"] [assign] warn_non_default_branch = true -contributing_url = "https://docs.dragonos.org/zh-cn/latest/community/code_contribution/index.html" +contributing_url = "https://community.dragonos.org/contributors/" users_on_vacation = [] [assign.adhoc_groups] From 6171c424025f3005af48f9668a1266a3925ab128 Mon Sep 17 00:00:00 2001 From: Samuel Dai Date: Mon, 21 Oct 2024 01:24:07 +0800 Subject: [PATCH 05/30] update rintotek message (#1003) --- docs/community/contact/index.rst | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/docs/community/contact/index.rst b/docs/community/contact/index.rst index b0baaa082..0e5295631 100644 --- a/docs/community/contact/index.rst +++ b/docs/community/contact/index.rst @@ -35,16 +35,5 @@ DragonOS社区的捐赠信息将按年进行公开。赞助商、赞助者信息 社区管理、财务及法务主体 ------------------------- -DragonOS社区的管理、财务及法务主体为：灵高计算机系统（广州）有限公司。 - -我们是一家开源公司，我们坚信，开源能为我国将来的IT，打下更好的基础。我们也通过其他业务创收，投入到DragonOS的研发之中。 - -公司负责DragonOS社区的运营、财务、法务事项处理工作。 - -地址：广东省广州市番禺区小谷围街广州大学城华南理工大学大学城校区 - -邮件：contact@DragonOS.org - -官网：https://ringotek.com.cn - +灵高是DragonOS社区为满足相关监管合规要求，成立的 **非营利性质** 的单位。详情请见：https://ringotek.com.cn From b9966addf1eba1c9feb528c2944ec87e58e393a3 Mon Sep 17 00:00:00 2001 From: MemoryShore <1353318529@qq.com> Date: Tue, 22 Oct 2024 22:32:50 +0800 Subject: [PATCH 06/30] update novashell to 7bb802ad1e (#1006) --- user/dadk/config/nova_shell-0.1.0.dadk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/user/dadk/config/nova_shell-0.1.0.dadk b/user/dadk/config/nova_shell-0.1.0.dadk index 0e9c456cf..b3cd99ec4 100644 --- a/user/dadk/config/nova_shell-0.1.0.dadk +++ b/user/dadk/config/nova_shell-0.1.0.dadk @@ -6,7 +6,7 @@ "BuildFromSource": { "Git": { "url": "https://git.mirrors.dragonos.org.cn/DragonOS-Community/NovaShell.git", - "revision": "b0dea7c16f" + "revision": "7bb802ad1e" } } }, From 80c9e8f8f0adb25c1a33ff09cb51e7df5b1cdd99 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=BB=84=E9=93=AD=E6=B6=9B?= <114841534+1037827920@users.noreply.github.com> Date: Fri, 25 Oct 2024 14:26:37 +0800 Subject: [PATCH 07/30] =?UTF-8?q?fix(tools):=E6=94=AF=E6=8C=81=E5=9C=A8?= =?UTF-8?q?=E6=97=A0kvm=E6=94=AF=E6=8C=81=E4=B8=8B=E8=BF=90=E8=A1=8CDragon?= =?UTF-8?q?OS=20(#1010)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(tools):支持在无kvm支持下运行DragonOS * fix: 修正问题：在非Linux上面加速选项选择的不正确 --------- Co-authored-by: jinlong --- tools/run-qemu.sh | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/tools/run-qemu.sh b/tools/run-qemu.sh index 8f201c8de..ba2d3ce1a 100644 --- a/tools/run-qemu.sh +++ b/tools/run-qemu.sh @@ -48,10 +48,16 @@ qemu_trace_usb=trace:usb_xhci_reset,trace:usb_xhci_run,trace:usb_xhci_stop,trace # 根据架构设置qemu的加速方式 if [ ${ARCH} == "i386" ] || [ ${ARCH} == "x86_64" ]; then - qemu_accel="kvm" - if [ $(uname) == Darwin ]; then - qemu_accel=hvf + qemu_accel="kvm" + if [ $(uname) == Darwin ]; then + qemu_accel=hvf + else + # 判断系统kvm模块是否加载 + if [ ! -e /dev/kvm ]; then + # kvm模块未加载，使用tcg加速 + qemu_accel="tcg" fi + fi fi # uboot版本 @@ -82,7 +88,10 @@ BIOS_TYPE="" VIRTIO_BLK_DEVICE=false # 如果qemu_accel不为空 if [ -n "${qemu_accel}" ]; then - QEMU_ACCELARATE="-machine accel=${qemu_accel} -enable-kvm " + QEMU_ACCELARATE=" -machine accel=${qemu_accel} " + if [ "${qemu_accel}" == "kvm" ]; then + QEMU_ACCELARATE+=" -enable-kvm " + fi fi if [ ${ARCH} == "i386" ] || [ ${ARCH} == "x86_64" ]; then From fae6e9ade46a52976ad5d099643d51cc20876448 Mon Sep 17 00:00:00 2001 From: linfeng Date: Fri, 25 Oct 2024 15:59:57 +0800 Subject: [PATCH 08/30] feat(ebpf):[WIP] add eBPF support (#948) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(kprobe): Add basic kprobe support for x86_64 * feat: add ebpf support (#912) - 实现bpf()一部分命令，包括几种基本map，相关的helper函数 - 实现部分perf相关的数据结构 - 暂时为文件实现简单mmap - 实现一个使用kprobe统计syscall 调用次数的ebpf程序 对eBPF支持程度(基本)： - 简单的eBPF程序(没有指定特殊的Map) - 使用内核已经实现的Map的eBPF程序 - 可以和kprobe配合使用 - 内核Map相关的接口定义已经实现，添加新的Map较为简单 不支持的功能: - 区分不同的eBPF程序类型(Network/Cgroup)并限定可调用的helper函数集 - 与内核其它跟踪机制配合(tracepoint) - 其它helper和Map todo - [ ] 修改mmap，需要讨论，因为这个和块缓存层相关 - [x] 添加文档 - [x] 修复可能的错误 - [x] 增加rbpf版本信息 * feat: add /sys/devices/system/cpu/possible file * feat: add /sys/devices/system/cpu/online --- .github/workflows/cache-toolchain.yml | 94 + .github/workflows/makefile.yml | 2 +- docs/index.rst | 1 + docs/kernel/trace/eBPF.md | 324 ++ docs/kernel/trace/ebpf_flow.png | Bin 0 -> 66484 bytes docs/kernel/trace/index.rst | 11 + docs/kernel/trace/kprobe.md | 57 + docs/kernel/trace/kprobe_flow.png | Bin 0 -> 47130 bytes kernel/Cargo.toml | 6 + kernel/Makefile | 4 +- kernel/crates/kprobe/Cargo.toml | 11 + .../crates/kprobe/src/arch/loongarch64/mod.rs | 112 + kernel/crates/kprobe/src/arch/mod.rs | 211 ++ kernel/crates/kprobe/src/arch/rv64/mod.rs | 157 + kernel/crates/kprobe/src/arch/x86/mod.rs | 135 + kernel/crates/kprobe/src/lib.rs | 7 + kernel/crates/rbpf/.appveyor.yml | 21 + kernel/crates/rbpf/.gitignore | 2 + kernel/crates/rbpf/Cargo.toml | 78 + kernel/crates/rbpf/LICENSE-APACHE | 202 ++ kernel/crates/rbpf/LICENSE-MIT | 25 + kernel/crates/rbpf/README.md | 743 +++++ kernel/crates/rbpf/clippy.toml | 1 + kernel/crates/rbpf/examples/disassemble.rs | 26 + kernel/crates/rbpf/examples/helper.rs | 3 + kernel/crates/rbpf/examples/load_elf.rs | 115 + .../rbpf/examples/load_elf__block_a_port.c | 43 + kernel/crates/rbpf/examples/rbpf_plugin.rs | 126 + kernel/crates/rbpf/examples/to_json.rs | 74 + kernel/crates/rbpf/examples/uptime.rs | 78 + kernel/crates/rbpf/mk/appveyor.bat | 72 + kernel/crates/rbpf/rustfmt.toml | 3 + kernel/crates/rbpf/src/asm_parser.rs | 642 ++++ kernel/crates/rbpf/src/assembler.rs | 277 ++ kernel/crates/rbpf/src/cranelift.rs | 1230 +++++++ kernel/crates/rbpf/src/disassembler.rs | 807 +++++ kernel/crates/rbpf/src/ebpf.rs | 635 ++++ kernel/crates/rbpf/src/helpers.rs | 488 +++ kernel/crates/rbpf/src/insn_builder.rs | 2199 +++++++++++++ kernel/crates/rbpf/src/interpreter.rs | 708 ++++ kernel/crates/rbpf/src/jit.rs | 1054 ++++++ kernel/crates/rbpf/src/lib.rs | 1782 ++++++++++ kernel/crates/rbpf/src/no_std_error.rs | 41 + kernel/crates/rbpf/src/stack.rs | 75 + kernel/crates/rbpf/src/verifier.rs | 386 +++ kernel/crates/rbpf/tests/assembler.rs | 655 ++++ kernel/crates/rbpf/tests/common.rs | 97 + kernel/crates/rbpf/tests/cranelift.rs | 2257 +++++++++++++ kernel/crates/rbpf/tests/disassembler.rs | 377 +++ kernel/crates/rbpf/tests/misc.rs | 571 ++++ kernel/crates/rbpf/tests/ubpf_jit_x86_64.rs | 2891 +++++++++++++++++ kernel/crates/rbpf/tests/ubpf_verifier.rs | 177 + kernel/crates/rbpf/tests/ubpf_vm.rs | 2670 +++++++++++++++ kernel/src/arch/riscv64/interrupt/handle.rs | 15 +- kernel/src/arch/riscv64/interrupt/mod.rs | 19 + kernel/src/arch/riscv64/kprobe.rs | 85 + kernel/src/arch/riscv64/mod.rs | 1 + kernel/src/arch/x86_64/interrupt/mod.rs | 20 +- kernel/src/arch/x86_64/interrupt/trap.rs | 25 +- kernel/src/arch/x86_64/kprobe.rs | 65 + kernel/src/arch/x86_64/mod.rs | 1 + kernel/src/bpf/helper/consts.rs | 11 + kernel/src/bpf/helper/mod.rs | 340 ++ kernel/src/bpf/helper/print.rs | 25 + kernel/src/bpf/map/array_map.rs | 283 ++ kernel/src/bpf/map/hash_map.rs | 156 + kernel/src/bpf/map/lru.rs | 151 + kernel/src/bpf/map/mod.rs | 416 +++ kernel/src/bpf/map/queue.rs | 154 + kernel/src/bpf/map/util.rs | 100 + kernel/src/bpf/mod.rs | 50 + kernel/src/bpf/prog/mod.rs | 123 + kernel/src/bpf/prog/util.rs | 112 + kernel/src/bpf/prog/verifier.rs | 131 + kernel/src/common/string.h | 9 + kernel/src/debug/kprobe/args.rs | 66 + kernel/src/debug/kprobe/mod.rs | 183 ++ kernel/src/debug/kprobe/test.rs | 84 + kernel/src/debug/mod.rs | 1 + kernel/src/debug/traceback/traceback.c | 13 + kernel/src/debug/traceback/traceback.h | 3 +- kernel/src/driver/base/cpu.rs | 96 +- kernel/src/driver/base/init.rs | 4 +- kernel/src/exception/debug.rs | 33 + kernel/src/exception/ebreak.rs | 37 + kernel/src/exception/mod.rs | 2 + kernel/src/filesystem/vfs/file.rs | 31 +- kernel/src/filesystem/vfs/mod.rs | 3 + kernel/src/include/bindings/linux_bpf.rs | 2430 ++++++++++++++ kernel/src/include/bindings/mod.rs | 10 +- kernel/src/init/init.rs | 6 +- kernel/src/lib.rs | 4 +- kernel/src/mm/fault.rs | 6 +- kernel/src/mm/ucontext.rs | 10 +- kernel/src/perf/bpf.rs | 333 ++ kernel/src/perf/kprobe.rs | 159 + kernel/src/perf/mod.rs | 337 ++ kernel/src/perf/util.rs | 123 + kernel/src/smp/cpu/mod.rs | 1 - kernel/src/syscall/mod.rs | 14 + kernel/src/time/syscall.rs | 4 +- user/apps/test_ebpf/.gitignore | 3 + user/apps/test_ebpf/Cargo.toml | 16 + user/apps/test_ebpf/Makefile | 61 + user/apps/test_ebpf/src/main.rs | 60 + .../test_ebpf/syscall_ebpf/.cargo/config.toml | 2 + user/apps/test_ebpf/syscall_ebpf/.gitignore | 9 + .../syscall_ebpf/.vscode/settings.json | 3 + user/apps/test_ebpf/syscall_ebpf/Cargo.toml | 3 + user/apps/test_ebpf/syscall_ebpf/README.md | 32 + .../syscall_ebpf-common/Cargo.toml | 14 + .../syscall_ebpf-common/src/lib.rs | 1 + .../syscall_ebpf-ebpf/.cargo/config.toml | 6 + .../syscall_ebpf-ebpf/.helix/config.toml | 2 + .../syscall_ebpf-ebpf/.vim/coc-settings.json | 4 + .../syscall_ebpf-ebpf/.vscode/settings.json | 4 + .../syscall_ebpf/syscall_ebpf-ebpf/Cargo.toml | 33 + .../syscall_ebpf-ebpf/rust-toolchain.toml | 13 + .../syscall_ebpf-ebpf/src/main.rs | 44 + .../test_ebpf/syscall_ebpf/xtask/Cargo.toml | 8 + .../test_ebpf/syscall_ebpf/xtask/src/build.rs | 42 + .../syscall_ebpf/xtask/src/build_ebpf.rs | 67 + .../test_ebpf/syscall_ebpf/xtask/src/main.rs | 36 + .../test_ebpf/syscall_ebpf/xtask/src/run.rs | 55 + user/apps/test_tokio/src/main.rs | 2 - user/dadk/config/test_ebpf_0_1_0.dadk | 23 + 126 files changed, 29529 insertions(+), 62 deletions(-) create mode 100644 .github/workflows/cache-toolchain.yml create mode 100644 docs/kernel/trace/eBPF.md create mode 100644 docs/kernel/trace/ebpf_flow.png create mode 100644 docs/kernel/trace/index.rst create mode 100644 docs/kernel/trace/kprobe.md create mode 100644 docs/kernel/trace/kprobe_flow.png create mode 100644 kernel/crates/kprobe/Cargo.toml create mode 100644 kernel/crates/kprobe/src/arch/loongarch64/mod.rs create mode 100644 kernel/crates/kprobe/src/arch/mod.rs create mode 100644 kernel/crates/kprobe/src/arch/rv64/mod.rs create mode 100644 kernel/crates/kprobe/src/arch/x86/mod.rs create mode 100644 kernel/crates/kprobe/src/lib.rs create mode 100644 kernel/crates/rbpf/.appveyor.yml create mode 100644 kernel/crates/rbpf/.gitignore create mode 100644 kernel/crates/rbpf/Cargo.toml create mode 100644 kernel/crates/rbpf/LICENSE-APACHE create mode 100644 kernel/crates/rbpf/LICENSE-MIT create mode 100644 kernel/crates/rbpf/README.md create mode 100644 kernel/crates/rbpf/clippy.toml create mode 100644 kernel/crates/rbpf/examples/disassemble.rs create mode 100644 kernel/crates/rbpf/examples/helper.rs create mode 100644 kernel/crates/rbpf/examples/load_elf.rs create mode 100644 kernel/crates/rbpf/examples/load_elf__block_a_port.c create mode 100644 kernel/crates/rbpf/examples/rbpf_plugin.rs create mode 100644 kernel/crates/rbpf/examples/to_json.rs create mode 100644 kernel/crates/rbpf/examples/uptime.rs create mode 100644 kernel/crates/rbpf/mk/appveyor.bat create mode 100644 kernel/crates/rbpf/rustfmt.toml create mode 100644 kernel/crates/rbpf/src/asm_parser.rs create mode 100644 kernel/crates/rbpf/src/assembler.rs create mode 100644 kernel/crates/rbpf/src/cranelift.rs create mode 100644 kernel/crates/rbpf/src/disassembler.rs create mode 100644 kernel/crates/rbpf/src/ebpf.rs create mode 100644 kernel/crates/rbpf/src/helpers.rs create mode 100644 kernel/crates/rbpf/src/insn_builder.rs create mode 100644 kernel/crates/rbpf/src/interpreter.rs create mode 100644 kernel/crates/rbpf/src/jit.rs create mode 100644 kernel/crates/rbpf/src/lib.rs create mode 100644 kernel/crates/rbpf/src/no_std_error.rs create mode 100644 kernel/crates/rbpf/src/stack.rs create mode 100644 kernel/crates/rbpf/src/verifier.rs create mode 100644 kernel/crates/rbpf/tests/assembler.rs create mode 100644 kernel/crates/rbpf/tests/common.rs create mode 100644 kernel/crates/rbpf/tests/cranelift.rs create mode 100644 kernel/crates/rbpf/tests/disassembler.rs create mode 100644 kernel/crates/rbpf/tests/misc.rs create mode 100644 kernel/crates/rbpf/tests/ubpf_jit_x86_64.rs create mode 100644 kernel/crates/rbpf/tests/ubpf_verifier.rs create mode 100644 kernel/crates/rbpf/tests/ubpf_vm.rs create mode 100644 kernel/src/arch/riscv64/kprobe.rs create mode 100644 kernel/src/arch/x86_64/kprobe.rs create mode 100644 kernel/src/bpf/helper/consts.rs create mode 100644 kernel/src/bpf/helper/mod.rs create mode 100644 kernel/src/bpf/helper/print.rs create mode 100644 kernel/src/bpf/map/array_map.rs create mode 100644 kernel/src/bpf/map/hash_map.rs create mode 100644 kernel/src/bpf/map/lru.rs create mode 100644 kernel/src/bpf/map/mod.rs create mode 100644 kernel/src/bpf/map/queue.rs create mode 100644 kernel/src/bpf/map/util.rs create mode 100644 kernel/src/bpf/mod.rs create mode 100644 kernel/src/bpf/prog/mod.rs create mode 100644 kernel/src/bpf/prog/util.rs create mode 100644 kernel/src/bpf/prog/verifier.rs create mode 100644 kernel/src/debug/kprobe/args.rs create mode 100644 kernel/src/debug/kprobe/mod.rs create mode 100644 kernel/src/debug/kprobe/test.rs create mode 100644 kernel/src/exception/debug.rs create mode 100644 kernel/src/exception/ebreak.rs create mode 100644 kernel/src/include/bindings/linux_bpf.rs create mode 100644 kernel/src/perf/bpf.rs create mode 100644 kernel/src/perf/kprobe.rs create mode 100644 kernel/src/perf/mod.rs create mode 100644 kernel/src/perf/util.rs create mode 100644 user/apps/test_ebpf/.gitignore create mode 100644 user/apps/test_ebpf/Cargo.toml create mode 100644 user/apps/test_ebpf/Makefile create mode 100644 user/apps/test_ebpf/src/main.rs create mode 100644 user/apps/test_ebpf/syscall_ebpf/.cargo/config.toml create mode 100644 user/apps/test_ebpf/syscall_ebpf/.gitignore create mode 100644 user/apps/test_ebpf/syscall_ebpf/.vscode/settings.json create mode 100644 user/apps/test_ebpf/syscall_ebpf/Cargo.toml create mode 100644 user/apps/test_ebpf/syscall_ebpf/README.md create mode 100644 user/apps/test_ebpf/syscall_ebpf/syscall_ebpf-common/Cargo.toml create mode 100644 user/apps/test_ebpf/syscall_ebpf/syscall_ebpf-common/src/lib.rs create mode 100644 user/apps/test_ebpf/syscall_ebpf/syscall_ebpf-ebpf/.cargo/config.toml create mode 100644 user/apps/test_ebpf/syscall_ebpf/syscall_ebpf-ebpf/.helix/config.toml create mode 100644 user/apps/test_ebpf/syscall_ebpf/syscall_ebpf-ebpf/.vim/coc-settings.json create mode 100644 user/apps/test_ebpf/syscall_ebpf/syscall_ebpf-ebpf/.vscode/settings.json create mode 100644 user/apps/test_ebpf/syscall_ebpf/syscall_ebpf-ebpf/Cargo.toml create mode 100644 user/apps/test_ebpf/syscall_ebpf/syscall_ebpf-ebpf/rust-toolchain.toml create mode 100644 user/apps/test_ebpf/syscall_ebpf/syscall_ebpf-ebpf/src/main.rs create mode 100644 user/apps/test_ebpf/syscall_ebpf/xtask/Cargo.toml create mode 100644 user/apps/test_ebpf/syscall_ebpf/xtask/src/build.rs create mode 100644 user/apps/test_ebpf/syscall_ebpf/xtask/src/build_ebpf.rs create mode 100644 user/apps/test_ebpf/syscall_ebpf/xtask/src/main.rs create mode 100644 user/apps/test_ebpf/syscall_ebpf/xtask/src/run.rs create mode 100644 user/dadk/config/test_ebpf_0_1_0.dadk diff --git a/.github/workflows/cache-toolchain.yml b/.github/workflows/cache-toolchain.yml new file mode 100644 index 000000000..4b6099c54 --- /dev/null +++ b/.github/workflows/cache-toolchain.yml @@ -0,0 +1,94 @@ +name: Cache toolchain + +on: workflow_call + +jobs: + build: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + + - name: Cache DragonOS GCC + id: cache-dragonos-gcc + uses: actions/cache@v3 + env: + cache-name: cache-dragonos-gcc + with: + path: | + ~/opt + ~/.bashrc + key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('tools/build_gcc_toolchain.sh') }} + + - if: ${{ steps.cache-dragonos-gcc.outputs.cache-hit != 'true' }} + name: build dragonos-gcc + continue-on-error: true + run: | + bash tools/build_gcc_toolchain.sh -f + + - uses: ./.github/actions/install-apt-packages + + - name: Cache build tools + id: cache-build-tools + uses: actions/cache@v3 + env: + cache-name: cache-build-tools + dadk_version: 0.1.11 + with: + path: | + ~/.cargo + ~/.rustup + ~/.bashrc + ~/opt + key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ env.dadk_version }}-${{ hashFiles('.github/workflows/cache-toolchain.yml') }}-${{ hashFiles('tools/install_musl_gcc.sh') }} + + - if: ${{ steps.cache-build-tools.outputs.cache-hit != 'true' }} + name: Install toolchain + continue-on-error: false + run: | + USE_GITHUB=1 bash tools/install_musl_gcc.sh + + cargo install cargo-binutils + rustup toolchain install nightly-x86_64-unknown-linux-gnu + rustup toolchain install nightly-2024-07-23-x86_64-unknown-linux-gnu + rustup toolchain install nightly-2023-08-15-x86_64-unknown-linux-gnu + rustup component add rust-src --toolchain nightly-2024-07-23-x86_64-unknown-linux-gnu + rustup component add rust-src --toolchain nightly-2023-08-15-x86_64-unknown-linux-gnu + rustup target add x86_64-unknown-none --toolchain nightly-2024-07-23-x86_64-unknown-linux-gnu + rustup target add x86_64-unknown-none --toolchain nightly-2023-08-15-x86_64-unknown-linux-gnu + + rustup toolchain install nightly-2024-07-23-riscv64gc-unknown-linux-gnu --force-non-host + rustup toolchain install nightly-2023-08-15-riscv64gc-unknown-linux-gnu --force-non-host + rustup target add riscv64gc-unknown-none-elf --toolchain nightly-2024-07-23-riscv64gc-unknown-linux-gnu + rustup target add riscv64imac-unknown-none-elf --toolchain nightly-2024-07-23-riscv64gc-unknown-linux-gnu + rustup target add riscv64gc-unknown-none-elf --toolchain nightly-2023-08-15-riscv64gc-unknown-linux-gnu + rustup target add riscv64imac-unknown-none-elf --toolchain nightly-2023-08-15-riscv64gc-unknown-linux-gnu + + rustup component add rust-src --toolchain nightly-x86_64-unknown-linux-gnu + rustup component add rust-src + rustup component add llvm-tools-preview + + rustup component add rustfmt + rustup component add rustfmt --toolchain nightly-x86_64-unknown-linux-gnu + rustup component add rustfmt --toolchain nightly-2024-07-23-x86_64-unknown-linux-gnu + rustup component add rustfmt --toolchain nightly-2023-08-15-x86_64-unknown-linux-gnu + rustup component add rustfmt --toolchain nightly-2024-07-23-riscv64gc-unknown-linux-gnu + rustup component add rustfmt --toolchain nightly-2023-08-15-riscv64gc-unknown-linux-gnu + + rustup default nightly-2024-07-23 + + cargo install dadk --version 0.1.11 + + + userapp_musl_toolchain="nightly-2023-08-15-x86_64-unknown-linux-gnu" + + rustup toolchain install ${userapp_musl_toolchain} + rustup component add --toolchain ${userapp_musl_toolchain} rust-src + rustup target add --toolchain ${userapp_musl_toolchain} x86_64-unknown-linux-musl + + rustup target add x86_64-unknown-linux-musl --toolchain nightly-2024-07-23-x86_64-unknown-linux-gnu + rustup component add rust-src --toolchain nightly-2024-07-23-x86_64-unknown-linux-gnu + cargo install bpf-linker + + diff --git a/.github/workflows/makefile.yml b/.github/workflows/makefile.yml index 03ddb7ecf..cfb550834 100644 --- a/.github/workflows/makefile.yml +++ b/.github/workflows/makefile.yml @@ -51,7 +51,7 @@ jobs: env: ARCH: ${{ matrix.arch }} HOME: /root - run: bash -c "source /root/.cargo/env && cd kernel && make test" + run: bash -c "source /root/.cargo/env && cd kernel && make test && make test-rbpf" build-x86_64: diff --git a/docs/index.rst b/docs/index.rst index be25163df..5d2aa82b3 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -31,6 +31,7 @@ kernel/ktest/index kernel/cpu_arch/index kernel/libs/index + kernel/trace/index .. toctree:: diff --git a/docs/kernel/trace/eBPF.md b/docs/kernel/trace/eBPF.md new file mode 100644 index 000000000..b9a1bf1a8 --- /dev/null +++ b/docs/kernel/trace/eBPF.md @@ -0,0 +1,324 @@ +# eBPF + +> 作者: 陈林峰 +> +> Email: chenlinfeng25@outlook.com + +## 概述 + +eBPF 是一项革命性的技术，起源于 Linux 内核，它可以在特权上下文中（如操作系统内核）运行沙盒程序。它用于安全有效地扩展内核的功能，而无需通过更改内核源代码或加载内核模块的方式来实现。 + +从历史上看，由于内核具有监督和控制整个系统的特权，操作系统一直是实现可观测性、安全性和网络功能的理想场所。同时，由于操作系统内核的核心地位和对稳定性和安全性的高要求，操作系统内核很难快速迭代发展。因此在传统意义上，与在操作系统本身之外实现的功能相比，操作系统级别的创新速度要慢一些。 + +eBPF 从根本上改变了这个方式。通过允许在操作系统中运行沙盒程序的方式，应用程序开发人员可以运行 eBPF 程序，以便在运行时向操作系统添加额外的功能。然后在 JIT 编译器和验证引擎的帮助下，操作系统确保它像本地编译的程序一样具备安全性和执行效率。这引发了一股基于 eBPF 的项目热潮，它们涵盖了广泛的用例，包括下一代网络实现、可观测性和安全功能等领域。 + +## eBPF In DragonOS + +在一个新的OS上添加eBPF的支持需要了解eBPF的运行过程，通常，eBPF需要用户态工具和内核相关基础设施配合才能发挥其功能。而新的OS通常会兼容Linux上的应用程序，这可以进一步简化对用户态工具的移植工作，只要内核实现相关的系统调用和功能，就可以配合已有的工具完成eBPF的支持。 + +## eBPF的运行流程 + +![image-20240909165945192](./ebpf_flow.png) + +如图所示，eBPF程序的运行过程分为三个主要步骤： + +1. 源代码->二进制 + 1. 用户可以使用python/C/Rust编写eBPF程序，并使用相关的工具链编译源代码到二进制程序 + 2. 这个步骤中，用户需要合理使用helper函数丰富eBPF程序功能 +2. 加载eBPF程序 + 1. 用户态的工具库会封装内核提供的系统调用接口，以简化用户的工作。用户态工具对eBPF程序经过预处理后发出系统调用，请求内核加载eBPF程序。 + 1. 内核首先会对eBPF程序进行验证，检查程序的正确性和合法性，同时也会对程序做进一步的处理 + 1. 内核会根据用户请求，将eBPF程序附加到内核的挂载点上(kprobe/uprobe/trace_point) + 1. 在内核运行期间，当这些挂载点被特定的事件触发， eBPF程序就会被执行 +3. 数据交互 + 1. eBPF程序可以收集内核的信息，用户工具可以选择性的获取这些信息 + 2. eBPF程序可以直接将信息输出到文件中，用户工具通过读取和解析文件中的内容拿到信息 + 3. eBPF程序通过Map在内核和用户态之间共享和交换数据 + + + +## 用户态支持 + +用户态的eBPF工具库有很多，比如C的libbpf，python的bcc, Rust的Aya，总体来说，这些工具的处理流程都大致相同。DragonOS当前支持[Aya](https://github.com/aya-rs/aya)框架编写的eBPF程序，以Aya为例，用户态的工具的处理过程如下: + +1. 提供eBPF使用的helper函数和Map抽象，方便实现eBPF程序 +2. 处理编译出来的eBPF程序，调用系统调用创建Map，获得对应的文件描述符 +3. 根据需要，更新Map的值(.data) +4. 根据重定位信息，对eBPF程序的相关指令做修改 +5. 根据内核版本，对eBPF程序中的bpf to bpf call进行处理 +6. 加载eBPF程序到内核中 +7. 对系统调用封装，提供大量的函数帮助访问eBPF的信息并与内核交互 + +DragonOS对Aya 库的支持并不完整。通过对Aya库的删减，我们实现了一个较小的[tiny-aya](https://github.com/DragonOS-Community/tiny-aya)。为了确保后期对Aya的兼容，tiny-aya只对Aya中的核心工具aya做了修改**，其中一些函数被禁用，因为这些函数的所需的系统调用或者文件在DragonOS中还未实现**。 + +### Tokio + +Aya需要使用异步运行时，通过增加一些系统调用和修复一些错误DragonOS现在已经支持基本的tokio运行时。 + +### 使用Aya创建eBPF程序 + +与Aya官方提供的[文档](https://aya-rs.dev/book/start/development/)所述，只需要根据其流程安装对应的Rust工具链，就可以按照模板创建eBPF项目。以当前实现的`syscall_ebf`为例，这个程序的功能是统计系统调用的次数，并将其存储在一个HashMap中。 + +``` +├── Cargo.toml +├── README.md +├── syscall_ebpf +├── syscall_ebpf-common +├── syscall_ebpf-ebpf +└── xtask +``` + +在user/app目录中，项目结构如上所示： + +- `syscall_ebpf-ebpf`是 eBPF代码的实现目录，其会被编译到字节码 +- `syscall_ebpf-common` 是公共库，方便内核和用户态进行信息交互 +- `syscall_ebpf` 是用户态程序，其负责加载eBPF程序并获取eBPF程序产生的数据 +- `xtask` 是一个命令行工具，方便用户编译和运行用户态程序 + +为了在DragonOS中运行用户态程序，暂时还不能直接使用模板创建的项目： + +1. 这个项目不符合DragonOS对用户程序的项目结构要求，当然这可以通过稍加修改完成 +2. 因为DragonOS对tokio运行时的支持还不是完整体，需要稍微修改一下使用方式 + +``` +#[tokio::main(flavor = "current_thread")] +async fn main() -> Result<(), Box> { +``` + +3. 因为对Aya支持不是完整体，因此项目依赖的aya和aya-log需要换成tiny-aya中的实现。 + +``` +[dependencies] +aya = { git = "https://github.com/DragonOS-Community/tiny-aya.git" } +aya-log = { git = "https://github.com/DragonOS-Community/tiny-aya.git" } +``` + +只需要稍加修改，就可以利用Aya现有的工具完成eBPF程序的实现。 + +## 内核态支持 + +内核态支持主要为三个部分： + +1. kprobe实现：位于目录`kernel/crates/kprobe` +2. rbpf运行时：位于目录`kernel/crates/rbpf` +3. 系统调用支持 +4. helper函数支持 + +### rbpf + +由于rbpf之前只是用于运行一些简单的eBPF程序，其需要通过一些修改才能运行更复杂的程序。 + +1. 增加bpf to bpf call 的支持：通过增加新的栈抽象和保存和恢复必要的寄存器数据 +2. 关闭内部不必要的内存检查，这通常由内核的验证器完成 +3. 增加带所有权的数据结构避免生命周期的限制 + + + +### 系统调用 + +eBPF相关的系统调用都集中在`bpf()` 上，通过参数cmd来进一步区分功能，目前对其支持如下: + +```rust +pub fn bpf(cmd: bpf_cmd, attr: &bpf_attr) -> Result { + let res = match cmd { + // Map related commands + bpf_cmd::BPF_MAP_CREATE => map::bpf_map_create(attr), + bpf_cmd::BPF_MAP_UPDATE_ELEM => map::bpf_map_update_elem(attr), + bpf_cmd::BPF_MAP_LOOKUP_ELEM => map::bpf_lookup_elem(attr), + bpf_cmd::BPF_MAP_GET_NEXT_KEY => map::bpf_map_get_next_key(attr), + bpf_cmd::BPF_MAP_DELETE_ELEM => map::bpf_map_delete_elem(attr), + bpf_cmd::BPF_MAP_LOOKUP_AND_DELETE_ELEM => map::bpf_map_lookup_and_delete_elem(attr), + bpf_cmd::BPF_MAP_LOOKUP_BATCH => map::bpf_map_lookup_batch(attr), + bpf_cmd::BPF_MAP_FREEZE => map::bpf_map_freeze(attr), + // Program related commands + bpf_cmd::BPF_PROG_LOAD => prog::bpf_prog_load(attr), + // Object creation commands + bpf_cmd::BPF_BTF_LOAD => { + error!("bpf cmd {:?} not implemented", cmd); + return Err(SystemError::ENOSYS); + } + ty => { + unimplemented!("bpf cmd {:?} not implemented", ty) + } + }; + res +} +``` + +其中对创建Map命令会再次细分，以确定具体的Map类型，目前我们对通用的Map基本添加了支持: + +```rust +bpf_map_type::BPF_MAP_TYPE_ARRAY +bpf_map_type::BPF_MAP_TYPE_PERCPU_ARRAY +bpf_map_type::BPF_MAP_TYPE_PERF_EVENT_ARRAY +bpf_map_type::BPF_MAP_TYPE_HASH +bpf_map_type::BPF_MAP_TYPE_PERCPU_HASH +bpf_map_type::BPF_MAP_TYPE_QUEUE +bpf_map_type::BPF_MAP_TYPE_STACK +bpf_map_type::BPF_MAP_TYPE_LRU_HASH +bpf_map_type::BPF_MAP_TYPE_LRU_PERCPU_HASH + +bpf_map_type::BPF_MAP_TYPE_CPUMAP +| bpf_map_type::BPF_MAP_TYPE_DEVMAP +| bpf_map_type::BPF_MAP_TYPE_DEVMAP_HASH => { + error!("bpf map type {:?} not implemented", map_meta.map_type); + Err(SystemError::EINVAL)? +} +``` + +所有的Map都会实现定义好的接口，这个接口参考Linux的实现定义: + +```rust +pub trait BpfMapCommonOps: Send + Sync + Debug + CastFromSync { + /// Lookup an element in the map. + /// + /// See https://ebpf-docs.dylanreimerink.nl/linux/helper-function/bpf_map_lookup_elem/ + fn lookup_elem(&mut self, _key: &[u8]) -> Result> { + Err(SystemError::ENOSYS) + } + /// Update an element in the map. + /// + /// See https://ebpf-docs.dylanreimerink.nl/linux/helper-function/bpf_map_update_elem/ + fn update_elem(&mut self, _key: &[u8], _value: &[u8], _flags: u64) -> Result<()> { + Err(SystemError::ENOSYS) + } + /// Delete an element from the map. + /// + /// See https://ebpf-docs.dylanreimerink.nl/linux/helper-function/bpf_map_delete_elem/ + fn delete_elem(&mut self, _key: &[u8]) -> Result<()> { + Err(SystemError::ENOSYS) + } + /// For each element in map, call callback_fn function with map, + /// callback_ctx and other map-specific parameters. + /// + /// See https://ebpf-docs.dylanreimerink.nl/linux/helper-function/bpf_for_each_map_elem/ + fn for_each_elem(&mut self, _cb: BpfCallBackFn, _ctx: *const u8, _flags: u64) -> Result { + Err(SystemError::ENOSYS) + } + /// Look up an element with the given key in the map referred to by the file descriptor fd, + /// and if found, delete the element. + fn lookup_and_delete_elem(&mut self, _key: &[u8], _value: &mut [u8]) -> Result<()> { + Err(SystemError::ENOSYS) + } + /// perform a lookup in percpu map for an entry associated to key on cpu. + fn lookup_percpu_elem(&mut self, _key: &[u8], cpu: u32) -> Result> { + Err(SystemError::ENOSYS) + } + /// Get the next key in the map. If key is None, get the first key. + /// + /// Called from syscall + fn get_next_key(&self, _key: Option<&[u8]>, _next_key: &mut [u8]) -> Result<()> { + Err(SystemError::ENOSYS) + } + /// Push an element value in map. + fn push_elem(&mut self, _value: &[u8], _flags: u64) -> Result<()> { + Err(SystemError::ENOSYS) + } + /// Pop an element value from map. + fn pop_elem(&mut self, _value: &mut [u8]) -> Result<()> { + Err(SystemError::ENOSYS) + } + /// Peek an element value from map. + fn peek_elem(&self, _value: &mut [u8]) -> Result<()> { + Err(SystemError::ENOSYS) + } + /// Freeze the map. + /// + /// It's useful for .rodata maps. + fn freeze(&self) -> Result<()> { + Err(SystemError::ENOSYS) + } + /// Get the first value pointer. + fn first_value_ptr(&self) -> *const u8 { + panic!("value_ptr not implemented") + } +} +``` + +联通eBPF和kprobe的系统调用是[`perf_event_open`](https://man7.org/linux/man-pages/man2/perf_event_open.2.html)，这个系统调用在Linux中非常复杂，因此Dragon中并没有按照Linux进行实现，目前只支持其中两个功能: + + + +```rust +match args.type_ { + // Kprobe + // See /sys/bus/event_source/devices/kprobe/type + perf_type_id::PERF_TYPE_MAX => { + let kprobe_event = kprobe::perf_event_open_kprobe(args); + Box::new(kprobe_event) + } + perf_type_id::PERF_TYPE_SOFTWARE => { + // For bpf prog output + assert_eq!(args.config, perf_sw_ids::PERF_COUNT_SW_BPF_OUTPUT); + assert_eq!( + args.sample_type, + Some(perf_event_sample_format::PERF_SAMPLE_RAW) + ); + let bpf_event = bpf::perf_event_open_bpf(args); + Box::new(bpf_event) + } +} +``` + +- 其中一个`PERF_TYPE_SOFTWARE`是用来创建软件定义的事件，`PERF_COUNT_SW_BPF_OUTPUT` 确保这个事件用来采集bpf的输出。 +- `PERF_TYPE_MAX` 通常指示创建kprobe/uprobe事件，也就是用户程序使用kprobe的途径之一，用户程序可以将eBPF程序绑定在这个事件上 + +同样的，perf不同的事件也实现定义的接口: + +```rust +pub trait PerfEventOps: Send + Sync + Debug + CastFromSync + CastFrom { + fn mmap(&self, _start: usize, _len: usize, _offset: usize) -> Result<()> { + panic!("mmap not implemented for PerfEvent"); + } + fn set_bpf_prog(&self, _bpf_prog: Arc) -> Result<()> { + panic!("set_bpf_prog not implemented for PerfEvent"); + } + fn enable(&self) -> Result<()> { + panic!("enable not implemented"); + } + fn disable(&self) -> Result<()> { + panic!("disable not implemented"); + } + fn readable(&self) -> bool { + panic!("readable not implemented"); + } +} +``` + +这个接口目前并不稳定。 + +### helper函数支持 + +用户态工具通过系统调用和内核进行通信，完成eBPF数据的设置、交换。在内核中，eBPF程序的运行也需要内核的帮助，单独的eBPF程序并没有什么太大的用处，因此其会调用内核提供的`helper` 函数完成对内核资源的访问。 + +目前已经支持的大多数`helper` 函数是与Map操作相关: + +```rust +/// Initialize the helper functions. +pub fn init_helper_functions() { + let mut map = BTreeMap::new(); + unsafe { + // Map helpers::Generic map helpers + map.insert(1, define_func!(raw_map_lookup_elem)); + map.insert(2, define_func!(raw_map_update_elem)); + map.insert(3, define_func!(raw_map_delete_elem)); + map.insert(164, define_func!(raw_map_for_each_elem)); + map.insert(195, define_func!(raw_map_lookup_percpu_elem)); + // map.insert(93,define_func!(raw_bpf_spin_lock); + // map.insert(94,define_func!(raw_bpf_spin_unlock); + // Map helpers::Perf event array helpers + map.insert(25, define_func!(raw_perf_event_output)); + // Probe and trace helpers::Memory helpers + map.insert(4, define_func!(raw_bpf_probe_read)); + // Print helpers + map.insert(6, define_func!(trace_printf)); + + // Map helpers::Queue and stack helpers + map.insert(87, define_func!(raw_map_push_elem)); + map.insert(88, define_func!(raw_map_pop_elem)); + map.insert(89, define_func!(raw_map_peek_elem)); + } + BPF_HELPER_FUN_SET.init(map); +} +``` + diff --git a/docs/kernel/trace/ebpf_flow.png b/docs/kernel/trace/ebpf_flow.png new file mode 100644 index 0000000000000000000000000000000000000000..fc46be5d2db7d7dcac4a8b21f449252cc4f8cc32 GIT binary patch literal 66484 zcmeFZWk8hc_CBoJKt-ffI-~^wX=%}+OG1!VVCay}fvu>3gaSjC(v5&L45Bc=(A^*m zIdsDS!+W#0dXD>?^Z)#Q`0Y>Z!FgicE3S2|YZ3TJMV8W|EC!9OhVqt)Ugo%=QGM z6<0SaMz;M_!vu3=GCfaZ{1n0lXde;YDxW z@^VCl`0h^%Lm&+?aJF+I?5O1!fu*|MOk!eU6ZMqf2qI{n6dYWPiW&9iHMNVx#I|rY zG!wWJ5tN9yTPBeA!^0oHzIPcsDTD$7`K*?5b>Zi)RUW{LF8=u5IguZ~{=drt8~y*$ z2;yIYM$2qtc%jELEt57QX`KvJ#SjYCTc{^~Y<>X&f(r1nG$~h8@W_cI5l2#?G@`pJ z-~4wj1aNVvA?1Nz)1y9gC;N;Ulv)jYA2?pW6H^VVeuMpEaW!!Z6~!4YWWPJzxYe1! z12Oup9!lMfZWmJ2%6g=)ldiDnxif(o&)q@p^x}@^KO-(PAY&jDUAIslWJM6F$;Jnp zv+YT>Hd9}#%{m3!X^o0f99zPT%kCHG6Xa;sQc>ot)z=1-vW6Z>-5k4eZGeovD(}~W z%edJkr>;t%o8Dz|zrRK?ro-vAH!ieup#WV@R+jZJAQP$D4BhRb^+U9=JcN<2y%B*&AT>IAvrY4a=cUO}Nul1ijhv(c^ zF5N)>9>fxaVq1;|68nx)P*bDGxN$@TR)we>m4{31Mg9z-t}Jue*r6g5-;r{INeMdF z)xOV~*)Og1RT3pCwI->FQ}up4PRJdUO{|1L2)=~%q~YWM4KwP)p#t22BqdlNfY+k0 zBsI9Ou+YJ({wT~!p{r)N8ao$2K28F?@6YB(YxzoTZtlgoMfFTAbff}&e{V(}rsoD{ z^IQG5Q;Q~n(vC@T*s0L%9eq$qL_N-d+f6KwXKP#>tDwzWlXailuA=D0b6f73St)OI z+0QC^;=$k?iCErD7f07Ei+VrVE6rsMQa!DB+lh$f z;@t+9YKtkdX_4!bL8wSY0c&hWCT!I9iO}OZi{5^=qq(EGKvHafzMk~G=l7cwC70EM z?8mPXlW`&&?eGXOpMBRJ!2AO`6#w#d&opA0SegEu?0V;H4WscA*~v%lhwD=Uq^#Fg zhl&g%*}uM^5xGMrw7U162sWEeBZ@ehiFjsEU66EF${Rc1oyu6{lB=7cl&BEdM+lE0 z!AI__9`|-6PWRS5d|0lOjFVp)5SVaJ_1c;Xk_mNw9ygUmclmek>X3~^H)+ka#~ZY> zldxRf-~Bvsjjv%}Yi zc+t(0QZ%1DXB*gPk!$6l-d#Bb$;pM3Lh7F_Bzk$s3uQJI!Lc-i(u`KfzdXMIZ6`j- zP>3A6$Go3+B1^cM++W8AF=zU#Xo#zeqt1uW=F3bs1eVz2ykQ*An!}lg`Jn0?H*|V4 zB|klVVY%^MC2__^%=cvS;RA&m?%4b3cC#qQ5KjnCjrAa1xjlCjcd{cOL?Q=my13w>WD@A3`O42 zG2RjhX$+#Rqv~@a{I(ww@E&+_wJ*04&7Zp5y8|HHItyS*DUcVbGQ0eZl#O36qy`%i zr)>R*zwFy6{fSZzo$1VXR}c%s8k3NU=M9(8C#V>ry+OSsagO!xp)|Aby5z~~8yu8m zgYIa8N*VDlMEL5=ie~z$S^C9k5wZ_{Pa6?xW-|e_Ry97gIjj$~VNO0FBMwwhgkHr|hV@OsnciO|kbrE=>_WV$LHj_Ogd37q0kl1gE_-cL+LcrOidp1MGPANza0T>iy63Q>>Wlntr+$rbt=(ZwgYl859`U<r zP+}E+#4fQis?t8!95(FE9!0Q}L0R2) z?wY|+ipNS<>bvBc)aAYnSn1;vpOyZKMXR|$x{|QOotSr$f+~@>HRY6BdYj~w z)sT~&_Q(%XLf5D-8Gc>>Xworc@vKZ|6epBlnELv4qLamvKV@ClR@=gepj|L#)DKk0 zk1>T?j%B?89?-HxrC{Dco)WclxE%}iJ@I%@tI{+$s4d>!ph+66n&ObR$XQfJzEku` zZ*VSMQ4)4$YHMLw1gSAkNhfoiBg%+6(z5)vh6Jpja3h;d-6TQX9^ zirXlVxK$?eH_jq6ZAU3HM+OQDGphJ)bC5>g<-nB?B+!IcY$gOLS1Db1QNdJ)fs|y| z+1`6$wf#=^SF$Xf(2FjI-aW2|13EHJv#1<1Zl7|4x(4h{4#xj_ERv0sVwywT&CJh% z1O4sNv3G6VvFDTaw~-lpCL~b=tSvN{cPB@) zn-DGOUuo){EMb)+)t}Qr9q6YhF!;rHN*UnLbV)? z%wkJrQ)-b4NF78&`T4R_s?xW;9N0-+)1s`UH~Ge4K4Zr4H~*-dVdq8lC9*27d9hM@ z7*@N_z0y&-&m91sH;KAl*qISZ0aR{1Ko?Yd=>5BCd#g#hwT{LsBa!NvD#TO69;xI| zE?`;ZMK$DpLkbAbcr)U}Uuj|_2g=91F&IwY( zt2C`&pR&qPNmuyKS6iK|el!k4zP-`9jp#!mU=o_Gar_0A2~|Ej_xp2HRrNF7S4wS! zd@n~?za{ha^fbV003|dL(IBP#JnuH@6yiSmC?;S9U+bw{+~+<-@#QRt6x_S^CLZB3h2#H@{bvsaE~HJhv&BgxVZiWy_d8J zv#M2JEJ}-`fD`5jOs{#v>R-TuCw={avRcQDkwP=TZmL(ez!$%eOVe!z-t9I;Qv3zIi*1H8RS|OGOks5z2-ZALIfZ_e$1b0|t3c;zPt z6S0$~&h#7CJ0j*M0!bZnv~x36xjNv2E4T~z!J;)D-@hD9eG?V1G^P&DVB7jbcKW+I z5s0XtUA6(*I+Xgope?(`?dD|3%+6FfwF3PxEzTuD@4`49gm)hzOjj5iWT@&kgO0fS z?0%8)LGEJ?R-AGie-ZWaaB8IP+6R_%qM+fQY$`0uA^3f!&6T3dj5 zzebTv80RI|%yC+avn&4TGY!aq*ooAtdd(4qoZRsps)aQ%nuC z^J!%I%ykD6OG-|LF?y2Y2R-k7Q`ok|FLQN`cIR7+Sa#jB-pn+GNHHS>^`^wukza;$a#9VQDZQjkY>6&n<1E%k~eEZ3g2}DfyZW8T1U?<;!cmHs}uu9w3HUOQ2 zXhh4Z9^MF0ErGW3+hV?);KWny9YGxc-CFhuXBd-8!YBJ=!v4c&RO1ysx~pS_lw^uM z>5Ah;4@uFjAIt#wFp~{k__)T-+xaG6^*ude?5@{tQ(>}a0cKXRC7d~Q8CL(gZsJ}a zfaWh=T<$Bn+|U@wZKR}sBcpLz4)jp(tsQ4kBEAy=I(+^ru}3=P1?`P5vfI!7L6Ws3 z$nL+<|5?hiCrvXx2=9J>Zk@x7-k4de+|P=GK-t zaTSU`=oc0n$x)&4_;&MM#vB2aS)6#4bG8Dl0~#riR6dJ*an5#W@EzS+toc_IbT;Uw zfuL)igCyTY0(E>D=OweM>y}7<%bAy^futuc%P|gCOWTX=8O|1c&7VJ}|1;m4fU|W0 z=zwZaRj!F+XQ*0ECh-9lYj|W=?V#Y@f(eN|Ki_E^j#d-9x}i+0{XORfDL}OrVxpae zLf0-UpW1BmZI4hGZg=Tf(Z>q$1&v>DCCB&GEB)~j*5U^aCw2UYhYcoxP}sZuH|r zES#5Jc6uMgzRs`M4?cSWC?(GhwKrESl}b*IV7v~~uN<}RR39Iz!3YDA*#FA?0Ra}N zzj7t{If)A2fqkx#hZo%>K#K!XwDA$>qt+0I8F6o?y4pZ`XwYoU8{Uoo0nE#rawHj`0n zvYwBVJ@vg0zDva2d`v&v&mThtcP01%cBBZuj#^gL(X5NIh)c*3;Q}P#L z|NbYg{MV!+I+D$Af#6rb{&M<-ViG7(t<Y$_YZ^jf^- z-CKK2Ggjw57kmpaMU*GdNzQn)<#BPNgTX{JmIW|&6)szC#NVplQ608`5rNO)&L@Nn zTgE_stmFJUlg-}N=-swtA>)PjT%g2}dUXv_9q%RgW~$`Hikk2L*(q{IM-01ELUnSu z7mQd&&DMghON_Wm!Kcc)X$;`ZprA*LDEkikB=M!KdOr~Y>Mk{EKezmw>h}WP0np;xg|(DoCJY zZQK+pZu;C@gTE4>S>bAai{+KccLN0mEfs-vb(4S`+eU}2P6%@wRP`ZuUeiU4}z)UWm|*7aTVHvoMp zJK1UHY8!tsgi@Mo)#~9_>K(l3Oyy<3wTCm;>JtI~ zq9*fV)8(H8#8<)NTx0C^lpD#*s}w!t$yWex=Hoz&Kuiq=(i@8TZe-u3r>_S@W*@KO z;5p-~rv*(rh}W7$RalnFerPc?8`J+rt0Av?`)^W72pH{0m4FE-X^WoG3W3af?WvN3uT z){mNh8)(f|WAsARzr5fS&vi=b71zPJjoS7}GW`z8G698dxA`5XBfzB>tfUB_Wz>Zb zKm?!qo|ju^G`{OfO#D5EuYfQ$JoZQ)!6etImRT6;(fTcB-zQ9cu0rwe?1Y?MEq-=}TZscY0ZR zd~u=QG1wjC#|j7ZTY`ANlf*evL-Py?19<2`qSNO@U2lz6rBy|lwg51@D=Qa(KxXYh zVrgUtTIT%`kGGV2`?~G-E}p-*FXoB0>JS@tnr~mE>ok&y)pjx^Mo)(>-yaf$C7*WV z|DN|KjO^gTpX|fNMx&JZlr&s0n!6smpYMDH10vK8==c}BmH=&eS8RHg$ME9=2lR)U?Y7w7 z@oFyxxBXA6j`kS+8g|rY+ukpqeW!Xp6gdi7w2%vAiFD`1KHLc+Vv!@7obTrN*vG<- zIy&k#JXWXXI0+!8Q6F6XV>s#g zmr=pP2$umcek5U4VUr1GR}mN@`|@QMD5uSW+?*eCCJwSH!PE)wW!vWN@Ez5VGj z+t*t&VNK{55R8Jp^NH48dq^d?@b*r}?X4yiBVL|jva^X5?$5E)mwRvecwm~I!bXQ% zGq#r}E7!V`@CVxN?bg*|=f0kMHpT}JS)}p&yf{$qM@h_OD5*z{#C^)& z)Ns%V8fd?s;mMp{ago=&olUZT+{>}VC$NW&HYtY0C~=>+u}Ky&-0SPUR%W0Z8grMe zm}4eg?w|Icg6=NIvAW*iPC5yD4?R9CEz~Ww0pm)LNKUC2Vu@BD2>8Y=F7_wJN_QWkH4 zr(eJ{QKas~2aV~UWH+d4wl0J!CrenSKsI5o^TO$UjwtPGZb^m?u<;pxrT4gPaOj=Z zd1ch)_&(PY1QbCjwqvsH)s8_qNG}f+HL7f<+U}@n@Elr&-kC3;%^%#SDV0D+kTC=6 zJ__ZxF;oo=PPq4k(@6;gQ4Ci{&s260&T+1JR*#HHF-$Z6#2EDm*2%5-pFzcE@szbF zms*&2NrQN#O!=J0BH+XZ%w3G=QTmlN+SjxS^c5zV9wbvifiK@NYmMSeQI)<4%}}AB z->Y;yI1l6aWrXM(P@J&O(Wr8KgUZ}h8j-Ar{sTLhS_3emq81Kkm1o$aQ#NfU!l7uA zEkb$yxtF8n$y%fMawWW`TJQtZYs1xPkP{K{v!iNo|-CmUU+EB;H{I_Jv>>&W`mPYjky6ABG~SW2+r?U;eqjLD~?324V*ix_G## zd<hod_qD}mt30rNlbGjd)gcG8|EmiM4U>YQkzWVw8@ zK=uS{mh4`X|23}4{qbEes5=5&DlC0lu9>&ekQxb^a_k=w+(ghr|9r1>8a4Nr~M)hwt&xad0Q| z<-^+$aeTAQnj&Ok^G)>O!t2jx&t67$QoF2Tx

5%#XiKwxA}n_&tF)y)}A0kPY=j zM9igobA2KYi$wJ5br=d4Ta4#DoOg}eJtFzNgmlk<@!z$}*VJ!<;n_qF5J%9Ds^ue}9GszHoZSNwHNEgYc%B z+t&1_A3w>D^a?IvfO>STPu+k2Zdubn|%;VGzd^sry-QJs`v1cr*`O^;j~U^@8l z7~uBCl;GzheJ?i9l0F?}ovC{-6ygOVy*(>uGU?oM8-dY4I!A(;#9+X=j!2H&HXgtR zXB!pfs6y1{K2FKYzvXV)Q7efm_yO)S*qk`-xp$6Lk{<5bY7`n~|8aqY)f>yvN)q_T zxvd(kRit`)T-}z+WSMQ_wdfFThkgT78>+QD89aZ%@o?$otpk@SJ4 zCvaid;-mXd|5SyniE$e3M2HXOTBSL(VmGrweW>a^oZ`#u7Pc1pWu8S7=1Sdwj({{c zIrCL9gjb6YL~eLmAmJ<9UZFsi-vKB80T}GhD7T(H+`aQ?3BZ3cqWAQ`G=MN0*fVnk zE#Ri`ZG&olujeOu>xH6C_WFbqwN8e0s-rF0?i)wStj9N6S%OdaH8PqCRkTf8qp}B^ z=(1SLE&&D6D;seDE(W;`5_{cry6(YtNAdM0e5w=nb%V)nN!;+u)e>_`A{+kL&<)Du zkh#qwH3_5CN4UJ8UdCCn3OvDtOdB)7ke|(c zB=Sa6jrLnc&d2*{XDe&M)$9vQH<@K9a{V*8s-${Fu+os^-w2B?mm0UdJ}_zMZ*l># z{CLblF_L{BG#)Um#17ZJj)2aU&H0tCu2haPn>Yu6#tw?<#{t26GCNX#Y42pco8r0M z_|@*F5JT*0Fj76)84d|c{35OY>K-|4Y^FbXZ)LRPJ7j?!jgM#u3$j)+x>I|g z_lmx%7~+0SksJZcNo%6F;6ZJnv@LfSeybGhg5 zC4vIDnAMQhmCvWXE)SQCezuGdvY%i}3LBj4EPT2;wlgqw$w1ynVH3w z-9nivC;RNazqO~5&=q~#gclVM^nPAXK3<~zVCkbV;Xr?4aP4q zA#{>SB0-!Cr+n1fRNV;e{7FW`Ic^x55{X>+uv3`_pYBrJ)*f z+d8%S`<8=z>B^$8?_YdF=?2MSf?L=aZ9v)aaFC*!4^VBa+w8q%a!3{3QqpGH9;Gw+ z&=hI<-wn8$C1S1s(mx_j(4w=z1E`#aKG|UaNbbv4pKGTnun!0*U{OPscs zj5dZ$u`#v*7*kOJ&sI+WO)?@bTOmY9*?PiSOPe;*`+>41upCJL)cVe!e@7hSgFu?# zx;BtvSX-_T$sQx=+X2wx!nzvTQUebMbR`IKVgrqo;HKw$tK;qQLZ=}h8%n+TQQDMA zF)GijJ#M;wwcZc!$f)(!?^^;Fe(+ZhxoIihFGgr$NU|-`6r-#ABDQ);q(jd27 z+J!81WsT^?h|2PxZ~S4U1EQk`Di?8A?E_`@3|mxmc?O=8@Lft;25`%vMVKn~FHMnI$NV+*<8C)Lr8_&zCg zrs&DCk7d$`x@s#@de7ag+QGq}*4CT0G3*rypI62e&y5$V$Ex)z_|h=?)>CcC}S zHtMLzp|CqR6BPU-CA~1291Tolur?J{{2mwVq*`+ys~ub(<4+dj#@w zHTUOg%f6#M#x51M%3c!j*qS?DF6jhC`q{)LGm!QFgAGqPHwlXD(b>zFcM&pS)^4N;vrUJv^f6(oWTYxwN?AQ!p zK-!d7zfYYD#3(R;`;Hp&#y)tKYPYo{2I(tkdok%yYT=7l$>|4Tgn|VB+iRU5hIIi+ zc)-Wd0>8ade;l+E8L)3m0=1c$sP$5ex^r85bwlG1WiY7DQH1|tSVH@h4Dl z4_9hv?A+D-zO;1MnDq`!qI0=rAb2Fu6hGiNp>&cGK$C$q(s&7Y4+PkZX?jN|WdGf` z!VA8^PIi^9!0zcY>wyJ)BI0qCU*jC_XF>(ZQM^#zou`_QDjl=ItOmnD%mWBz?&!n4 zHPv`QPJ=3Eq>=K=KmYu*z@XZ&%DJWHh2jIa17rVX=krb4d0k}m3{{E0XuXACZQqH# z0X=9RT21Qhb@{=<#GFD%!zE@^%^g|^?+WL|9-HA}3SY*9SIOaG4s|}q`s)om*XU6d zVH}7f_3@zxQFprkamh5Gnx^5aDhD!aPfhlPp^q&<#N0*32)dOv#i=hZ&NaTh!H2uK z=>a#nc#V30)V?lOyIKrFA>_D{)?`o2IaK%b1>y{GH<--)P;E9uPXqa=(_C8$ZOoO; zxprDtt9(SyjHGXJA0Bb7EHKZ}KKdO2CI4OA=7z_%aXJEaSY;9lq@QEW%DoxcR1*cm zC1AZHz>f9jM2Q}8=cI;a1ncLCDar<<{Ygs%c8Mc$^2FZjV6*X__+8HuNG#c0vM-K; z*Axr}E(h1Sf`(ModExoWh`^Rr`Aasm%-glFssr0Z2?H=9TX6j~jhk+Le8k4A9QQ>0 z!HvE=ohjtbQ`jsBQ)b1RMEFq`L;B5$Yat*H7dg#$PzyVf`}SV~%I)KJp~;x&s#~Xq zSgY~C+^l$px(;$z3Rt!HIkn>+2Qq(8bVv$DDu-S<4nA7m!OOX{XDKQi&nPj- zf04GiV?eDh9dr{pdH3a@kuR(>NqjEd4qpu!a@Fv&-1s#>2z=na4{;X#6&5@qN1zV} zO;0FTKJmiMrYd2sj~$V61=5$&L=PZ z9$~>}$=c{*?>_s}bBOZRZ4H)cFbjj}K{}(@&@ujmbfJ_2?0@A%GSE!aIn9T9WaCMJ z=&Bq4EI-a`ew8xqA7(W>0!ou0vd%)%$cg((xsv)2BHL-2gx$;G3td)^{<=t{67PVbomp&$i-ofr3j zO)GCGl0&;KNQ~z6T?UIxFmr$i#QWzJo|an8_L&UK1C~5t4KbdvcD|dD8AZN0Sy#7F zE(K8c+C;6j;mn0o7xGn2(kvhHq4UY&zOVg!G?;>hQUIIv>>;GH%zm`&bTo2u>B+Bs zCZg2Q4Wr%1giliCTEZ`!22~n_;&pbj^UBE1)HnY*>4~#y$l73_Xgl)RvFcYyme#&& zfaz29tvut-k3QV)_C25XYpp4udqq`pb)v=r8`E8@=liG-tVYR4NQirwn+D)jfSF3K z+v^1n5-8JjSn@{nm+$p{?Ho0iQ-Zw~=a)F@jK<>xdd<6UD220y?O#l9fiv3|sRtzG z6ecHEBog{ku$D9YN|P~rBaz3|u5F28*}g#7%C2=Mj73@Dz{Fx^+-^ih6G@)W|&Nm>*D#w|^JAk(UyeC+O3}%s} z8xa6HrpkS)SAQkv`WS{qa=ZVhHd zmO7(+OkX#6btcq@Q{@89UyFH9`E^nnmZn`Fml<=#{{eJ+ten_k`0Z5`(efxM^c34Of3_Rh$aQEa`R zUA~>_F&gG|tGxH=<{=b?DMocZn_N%ca~mNfPmX$PzUtjVRYb@3rbkQ`?oIgUfkS`K z=65PmaH0Qev9Oy7pqFq6#ny2|CMhK)C1{X6$f(K5-KV}@*t){~h7E0K)(MyQsl#X$ z}i&+M&m}S^}zG;t-0x;`;^b$;S1g zP}BlAVxV^bGK`9Af9<{AjblR51F&a;@ZHpJ5Vy}@<=$K0+dEmUUTfp1p#=`8!*6d0 z?x>ma>i>1ePAGWB%_QwE&|YuxgJ5e{zi_2_O}p~z>z)5JHPEMU%ENgkB|W%+3L$#0 z=Iq(CLUvulA-WQ&4t9!4_AhkHKGsnPiO zptKmu5q(DoVbKk4b#<{%ev%Lg|XF11W&@+j`e8r3AND zfzs-7E|j4uinDjmkLS++GA3cVd1sO;PHRUPhW+e8NefavwDmfBn%%d~*BU2P0ss4! z`l=I|bESiZe4X{AAdHr2zj^tKlNCLgJcq3LpWhlisirLG{N>01pYiI-r`A_y7DVg{bTD=~qF^&bq}f z_JR$K9q?V^bTJ8i|3{_cqTWY?|LHOi&+UaHh<;e zKJvA-`LbxKbh0DkU-6)t2HM3<2_b-s=Y;RxhdbbN+l|hhW>yUrGf=M!WvO<)vz5#? zNzUXKXa$vF^I9<({wqM3{3|;EV7JlG=(mYi@@&f!dnc|fj&fA)_QKr zit3fw{CyL$YJOYxIt>}HEdUeZ*E9+i#MG;qZ4uVBQW>_-*>3AZ_bU^Qf*Jd-Safb@ z)KrlA_6VdZZS+VtsDbt&_u)Ii9wg^dbp8|e)2f6g)yWJP*!LP2-p<&M7geqw4-jxX z&L!gZ5fi2*djIqw>VViTT-3hfqVm!Lm?^7C3a0W9K`*|;X&N@<&;)Jk?1i(T2Y@!> z;9x#^mN4<@`uF-d%Tsq@w7-DY^loqen%e1N4vb&~?5XrS_kME)dquy>nt?D10H`K@ z#IwoG2Kldag_m9mcJQcVf9*l&S)<(-_6}!_x?di>-jsnxWKo?{%Q*RDxXBP@7nj>v z6%C;%AMi?Pn{Es`y)hV${(WO^NvokpK6OXOM+K|HsoeKRfwRPc061Ng`56FdUO)uj zlfD3vm59`cO$rt>*ir6SsrDr@m7cRNmf1g;4*oU>G*E) zNqBOH`12g}youf}?Oo%Ik)26g%9>FHW_h-AzA&MbRdeN|N2gD|Bu$D;uwAXQ*GSoS zT9b^ouxl|zPnLb>`>V0M$DhpC#Btrm`(%PPTW9Hm?eJQb+9>bbzqN0?BHAdVi0TF z6JDO>%=aOO+IMt?N<2oJeCe)H_se>-ivz+OvGTFdt;0|--Tb5j=D(}|$qZR^63acF zc5lCN$vkNMZ7R*~?4RRJLKtg!^#dWLSjjT$hLOqk`s;B%KNgpz zQ7Qc?%&U7~o~BiIUhnUS464InIqu$lgzI`-pg$}|&^i5BZ4Q$p?thcp)y)gQ-1(*I z({2z|D@J{NhG)K^l8Um*W_btq}0P{iH$cnpuNu}nTmy*^9EJ%Re9==i9XcxT~8 zymn%(ewCH-LH2a*weh{N3j3-{JZu@fv(2?j*jg)P-JHh7;utaiVK4ADSz3lpEkmCj1l8 z4I$S0i%awKVdbxCAr#s#f24amGAJTJSRHrA`5>*bZ5@r|I#@r zsa?;|qV9ikfh&6;Eq#$;;xH?oyb~@See`9%``qOX`#2??zPQL|(G5N`dF~0Z;zloC zLOGZ&DeapUN`SaA9?KLVDZ1l=4VHK}tyb_-IwHgS=(@SRe{buqUw~p3i3Esbf z=tWnx46A&FP4j2%9n46Frq|NqlcsN`{tg;;6r4?i-xkY1eeT2^cW?y}B{tiO_asEO zPh;twPxWBur0P{ORJhb+{9Saa&4$8ApOKMhRNbSA{mat_SP7gxX^}Y)K}y~CMpFdr zC5;^c#fhVGl6NJTW|Ye$<}C8Pk$g$4nWe5$2L~CsSw{-!F|Lqt$1qtMA7l>=RBQrW zWY#&kCLP3B^=*1WPCMvY0N@k*kFhKHPB`7*;fTRjs_XWkksV7a-DXXpv{c)?u{ z{)`Jiz=YGFMFT}YQmE$3moKB`_VLl)dy$?X z39nM~i&_lsc;$8&CVlcL*VV{gMe}h4!t4$<2!$)=k;iBGqMHdC{W@RYnr9FVXD5hj z*1i#_jCfMk-8pF#+BxLX=gzd{*V&9y|D@!B&VA2fKaqUT;|fobxUbZXQu1SSpK#Ia z!LkSk*4{E244-A0Lxah`fp_OrHc~J!@nn!6msyJuba?ax$SvbI&9#K9C2g!V1p`yq zQF0U6M#o))Q-w>yzT(b6X+CUpKG%&)D_+xU&!OaQ?G5U&nn_vuYPg9>4 z^IzaRAo7i54WQMYCS4@p4(8>HQW|qGr5Ul#G+gl><3mwjzJmQEb?#V48we~yavqaP z%F2_xk%WGuN@=UIFEDRQ>Saw~FwtM$T8LE07*=JG@KG=+MREqCk*v+Z29^DvBcCG+ zQXQpQ*!y?1C1HZFumo7DnSp?XSU5;{g0`C*6@@dJAsKY!O|wh+e~>Z{sQUK0G@P$& zna9^^2@8{2nhjW7j zzhjwR-rAZ-)ft-Sn#8B-nk+jR>f#lZ3<@4h*&XB%~X-wp2aJfqMI$emm`=2LUTb+ ztw#q-{#s5Y5RK1bJC@tITIIuiz%kY7d6V#7W0T|hVi>$IkFI4vvHX~}w%0o)t)=h# z$d>jj%Z|iyo)PNJ$@qJ)5MMV!yXl_hWdy601)R!H7GQB&jpYFl@mYvuqb@vrW;s@j z8+b49H9dar8lB`P)8VFA`e!L$YFUM3E96ZVc)q5+<(($n*&2MPT*B&mZ-I{c$Rl4q zoJGqINZ}$27vz(8WA~qg1hx03;nx|pL<;#kumUmdzVU3HkNUHZ`(;=$49l}t4?_P(19R%B#27 zw)KQbb{K~}7t5$m0Ri=*__Mi=TsIdqvmCt{$ABIEp#8}Kw~NGg>`GOhxbNe->x2>i zhOr~TD{JVBt%vW@_T@MFEBBG|fVm_Ynk5upalj_F@ei;nsO$|x1L<>&QaJxcK^FnG zh@yWI){>_1fYsOCQ)O&`hVDeRz3qd%)m;lTweXPF_g)QRBX>ji?Hw5ho=6Fk5k5(} zftBvUh%r~Eb@oahpl)(BZI1E5P6X$;Rbuv*_1hqt7L)Dvp3eJ4PL-`E;FOS2Cxvzf!0~`!e&~w`Y*y#MJjap@5y;xQ968RSsuiTiy zF9=CfiptX~F*6@1=r8BR5U6Fy^b)?UJJ|0I^NG6IJKd}DT-*NNlqJjT$bW|Hrf z>@U3DEX8D(OOwI#cMCR{>XVp$1Mw77>l^20-#zTtjqLlterMWx*vU9z{Zz?aCSyX6 z;LAqN+K5!7{?iA@3hESTJb7-WyyWc$??>V0Dd++am z9X@;S70-Isv-Vzb0Ny%U(*;se2Y-h%<`OK>w3v_KiR#?WnV9+cr7G((2OJHTMx0*1 zblyQa*`dY7D&1h=B*kp;O*l=I~Iy@JIYd_TK}VPaRsBcy}}g%$={}^18@& zBK)EP*4tkK-S|Ic54T9Hb9QChBOfqxApeP3?v>_S+;C76M3 z9~xN5rliGY+xYPzB6N#5kyTyanO^V=bA{rF&1mzX^RQt(t(RUyp8RRvHlR5yUt}8N z&$oSP7n*Ma2Xb)m@BK*t?b~)cLfod{M^_V7R*U#|IbRZ%2_rs3UBNi?$H6b`lAhDd zF?U@VT!!l|n$5=d^GTMehq_eAGWLH zX$;&OEOXl=e2jTll~{Xt!`4q4r^k3+ycJ}H8qH}|iJ!Ss9vsMnp8%~1d67-s{axwP z>vn%q1f=N!@M|m2G8O;Wj@|sA>(_5c%^|hIJ$E5mRe8r{@AkH@^gk2uCyk^UWzNx2 z%9h~rzu*D8Lfb>kaP(j_Ry#tAk%(C}uKPp!n|iz0H`_0K^euI#zP4Im=Ez)_KpTJ- zN3uxc7h9VMhI>tCn}V-|6rMVtTJNF1M?+c+epwdsAOyyWXvzEtyXEjls@eX9x1qa{(rf9P^Ic~PGGjPInLX7Y z)BjE0Y(CT*+>>ZZZZ+OUMeo2c)m9LWMeHsOJHmq~*XB*@JPKFFk#jA{qFXhN4VWLw zsPfpJzUNGvDuQUa^PHZ7Sk@$a_wrRMo?f2IoeD`L>s!SLxOA&5so420+1?s^t3C9O zWJvoV22(#26k1ph<(_Eq9pnmyGDt?r^E!ZuYSC$W=RoA-1!Hf^K>E}ZIa_#`GK zTD|c8X2*2y(sOqFrspL2GdH`|V$#Y`vTfH`P5AUwks0ln%rM<649VX7=uqbRJ4Eca zg!`$VOw4p(%iEIcW#EbtesGVthU(cb9_tA+pdSafh26L^fL1+yQuXk$TR4=SaBq@k z;`7Xr?Z!~3ZT`qNU(&T=3*HU0(E`HS;zibp^b(6LotpYoZHGWe zCrbrB;?&AmHLo|Vfe*GaDLdz*P|vL<8%H*t1hM3S9iL9F#gHBc0a&(YlPpb zn|rsJ?_7<@6`wy6db4$y)G*MjYs}n7C}`T=`EW(#Qr<91K-ov)&>5AmKpHO>SPGa@VegtDR zm-^^f&v?bwV7P4Ag21`-sU(99R$!y*=3ew04!4L<8FIZQupYd-3Q^Gc3yanlc0uLi zQNgj$&OsjMFt#wF@~=CEIT09*Vg3}?IUqvQU(GhmqVf_6>?j{UZHT^n{5b<`S}ejv z!r7rO?;9kaf}&E7{&R>`?DS2ENQn5RI|Gi+5!onN3oJ?zkrca{ZF>=SE&C{&r~3`T zXh8Y6Mtr3?)ahK{@;YZndVYHC`=Pvnofo>yPC;9%6Y0E0c?$VZRr_PzEG7y^cKhyc zKcBfElC}KAu{CP-a*D-IvqkXrWu|SC7z|;lp+AU%7YNKn!n~N(o<$RpIDgug#Ky?z zl-_}*ydu~0+>(`u0S;EGk}u^hrT}zlK!S+oA3|~yd7$>qECRx&@g-V|tn|~^d zPq}o|*jwo6jLKJKeGw;Y=#J^w7ZrLvHl`UGeY5#My@J6c@uZc?MXJ*u=F^0x5{e74 z+TKm1$BuQiN=sE7?>+A?$`SB*uMVBEac;L%c|%R0qgX24nm&RrtyZcWpS>O67aJG- zq)RJW3x4DEXr0AkPV4IGY9?HjPCo029afg$iI}fAY#=HfdpGoy1tZ(!N8+yn0!>!_>gZSD}Lq3)tuAyqxjTE_!9BDUNfbzO(WZGBJ9 z`1>j_7wH23n)GOhx*yD#L%e6Y`p$z93pv>W)sP?Q8pn~P(K17W{qqhdmE|^N1hB4q zPfkwvBohJ^+P|&dw9|o4eE1}ixy8(&XIYGaep5l@_%g%P{$MrolhA2R!1D(W9#oo7 z-hV*BiIIURhh`Z9oemEds7OL^d)3a<*Cv!(%B`(epPSgwtuVp-t|=~k)phbit3C+Z zSU_{V)cd)AGO>`X1id^~swv!gv${?Z1-|@J#^jc3m(| zLBAzUeBwavTZqMq3Vz4I^7v4P-81f{t3kfkY&KILD^zKKP5YBR{t0h<6&O&tS}R(i z^^I_}+(2|nyeSLjfP#e6v7d9sVNAZ$NQCM_&HIl}n1CNf{@rhcrS#y9@y95Lqh%OL#y@KCu zAah2mIZ3#hBM%D4nZ7;su|)&#GjF~(<{Q`jGJ}^+$9v4{@39p`9yC{U%s16mstfiw zcs7S&={Rz>T|Xh1q8YEEcyA0*wi+90c2Z!1DueUdoh zuuF2(k=1~n5nz2CLdg^YO>qm!ifAW|-b%zx;M-HQWO*sje_<7Ee&txTHexa}CCYmt zyAgg8PJJ!I`;sU(t*ah!*zJeGPKo2a&PP%mZTf6nl7iTZ>im1qr%l}(v*s_0P_N(L zY$8D$tGbF&?TnPGi0%m_xRcFHHomcS&xX@;BeFdqP+=cmVBGPhsC<nHZ$Vfo7QR8}5X8;6*rjuXf(-DPQ7;7PKkO|Y1Ml@+5h z0YNm7AE2#;Ox!~Lg$eOk_N8vKq-U+}>_hC)t88p*2Hj3JhflaAx&`2J4au3D&&@SK zCXQfLAJt4*|39l?G1R>R9%i>ZMl-WXSV186K#XwaESzc&Tq%r2%w}j}abjD&IqXNV@g%2I9(l{`3Z(8PVOPCvm_ci9TpZ;1+zOl zP{qQ++O%1@&kH=M!y(%0!eqd;!>UH>ECU01k;BLCNBhXdIsxP*K&nOSS4ufPBC zqd_2Ar+6Z~3kiV=Q0P+YnJ06w7#wdyA3T`wM+yA;)p+d|meb?}A1zhUZB+#jKGRVC zb)?q>@P66(op~V1%vH(n9~d}WVtC~MjMDaGSMMDptPp@f^cUoM;3tmrG$-LK7tLyS zB{BS~C`BgjVCeI6NC3=1m|h*(`OPL*Hz z&D)Xdxe0nm|B1CtK@PYa{hQ&LtQx_7hv z#j+$zMu7GnkRFCI&S>q#HTCwspRBfVy}GC@B7zhly-^?&lfRma3y6PSZcKHNaW6Db z3Bi9V5-NNXO0EcjDxm#NO=*rtDpyV!FYZ!h$Ff>+oSD*3h`oeoWM)=XRmH}|p$Cag z*EuKi*gwt_LFYi8*mkBAqodf+xPApN-*+tpSPJ#FH zp*CHIgtrR7eSuG*2>X$0O0Fm-WoW=?rt7ncSW((=Q6&g>v6oj?)KUdeH^pB7wPLIW z%=Ti-^$HhTU>^UMkylf(p`?7d_U8{VCx_?dcMKTtehyb$>v_ans)y2s1(17L*>gL) zquqroXQtkmn3!+X)h?IkY8)t`7Dy{`1J5$Z{j(1#Up`#s1!rZ^IM0R;<{8<|y+%{N z<>nMp8exBSn4g=Q>*a-dgqPBXKqLdfHPFODNZjZiBNq-%5(R{0d(2;_g(7#8qQ?%n z-{}NxUtgbQh4pN7kFYr#E;;Hvjo#)YjQzvxOCcel%~09&>}+sNX}p(~lF!XHj~A$^MeJyRmm4`x^8A}y)Ec8v}V zT?X!Sy#1rGG2P){)#Em&I8cm5F{&RPZm7>Zd`X1#&}$$_2)IYI>-+^y-O@Uzrwa0u#iZldm50|jpAAn=)~e*c_7H`B z1~Dg|&CKJ|Csz=eNW8_E|I5~+&>d|hk@0>l3<_d%mj2n=*{M74%NIK1_;e1l&2E=T zug>*~jD%z&SC!e!j4)V}k}@6WI5g{BG&nFr-y@BJZbD(vO0XeYG8%<$!U@ESN9R>2 z3N*ogMRydGy`WakuUCIVC>y%bWBE@3X)`XKYz@uK*PiYq_H2UCE8rR zfoo$N&ki>p@w?z*3sF7#1t5weR>o6r9F6&sdvTmrR~yvU{@WW-ikzvTZ6hr zFot^{K3wMHEI-_RvwQpwkDJ6TAshn|9uva>WTxo&^k|Qw30YWJ2w9Dl1O+|bKTP96 z9$6~D+u9-`6aG{lc$?p3659Ulc;v)Myr0!z`ud4nLzjEDN7G8V`p=AJ-F}h>AGtP{ zN}`ZETGHu(`2S(1!He*jzS37E@#DLlzHxD{!2w{CaJY#(VSvC5HmgaY1q3YXOXuZ& zUKbQRl>q53B~a>vY=kWJQRY3zT@&8Sa8+s667Ay)=?@MKk&?Q`a_WqH{<*O=CD`jm z5?TvxfjTfTG+bWpOYew)P&$3T2nQnv)m2rz_PZ}R9?Fj)1ErjACMK_<{w!R&ukh+t zN|8dElo|*JMt>n76qK}&=TG4}YYjo9@hhvVy{b5|@`{SNY6Iq-%Pwa}`E`zGcrX$a zI%sC+K(;Ix{u)ge#fgnB&nhl2Pvm#K1gY=%MrgyYEDF3w&iEg`0%bl3L=+$Ik1}x_ zDX&?HPoZJb5)n_=CaRjln&;X>5X?QLk~f4O08hF&Wrad57dvCIv9Z5?{o2sl%b%l= z2?qpWczC$imVp=rIx>*}7pG(WI}M|t>~@#ZvVe~y<7yfmtLAYQZ|74{(Mw2p&8z{X z;Aop`PdBu6Px0M8s?3dKP!IKa>4lEJe{kT16XBZn_3LX;hUs-g&D6v^zW>|#|NZ&Q zvSh+%r+I|jiI4ac{3r6t8Pfqy}%{jx>7+HXap1qM1(f zHJ*WPe^j#{A;u%D8W50eHYeR-Q#8BmQ+7Ex9%2VqMlF-|s_O&8I<@L|b$x~}Rw6A9hDt*I6tF9*gqPb+NJyl>rd zXb`%b9vK@0BO#YSMf5F5Ar$MsZPt8Ceu89@mWGBJqX8}< zACAJ;oCm&FS-Gk~&X*r!HBdv)vCF0QK#^!x0%hBBJ|IaUc)5m zb>2z)a8{$5$ZhqrJ$?DgSym5$iXI2{a<==-!Bm68=~W?@%~ufr(1Rb~;^L-vC#nOq z?=g|YW)`=7aR<`yp9q@?V)btXiQ@M1_JHLz8BnX{OkP`pHzQ^~t+n$`q>B<0$M5p<5_o5U z57y)zx0rZJ3)yJ{@%yrEHf%)f9?pN;VXBU7oX?-5&yTj$d0nTLr)tL#c}CfVSK%vS zCBdPg?ui zysInh^!T09I%o2yIuq5xI5vIjdJrN;$JN(XSJAd|Ow%(n#j)W8M{d7DnNgZ*M17Z3%61Boh5s7;RLm6aJ)I=?AoT7nym zOoqO?ms)`fDH5_}2y1kAj08jfrNaB|b%kVYWK{cpMtMb6j$O4Z3&Le9`PJ9;S$m<%TISgDXt7OTk^KC^jT zBd6RhwXw0WxA*CT2a6!;tZ(Flxt6gn^7&DW#MbL0o$Wa`(|n}2v6uOPEwiMbNyl-# zJuRasH3*F1x7eQ*o{M5k{MKNz`!!N=Qbks4j!kDHE_Q8LU+@yJit6hAm3beZpNU70z)xSFX19#(en3Tq$^u*kw64fF(|T5w2ZD|*B*d|w_;!Q{{g2H<(cJ|hmpM6E zQu>bH=UwdW@#QM#9XP(CK3I*Zp{&7D1rVcuQSPYIZY}|T=ER1~?_5V)Th~NA)qW2Q zJln=`>1|h+KWlgV;WhH9C#k8;?2zPt8N9&M$v(x-%G@$BLXfG26rI0(&G7r8^HeT~ zT-NHm&V3CZ2C}gTpH)w%xXU;@*e&-jAlS@^SdA32%_mpEY$(voeRVqN&y>JtfA);! z`5>aNPeoNVo4F$mWC@9uxR{5b!pdKj3PiBRy|GBgD$Lf`Cv4k(l7-V^y(#&4z9_wi zy>5qraUfAMf3^H#KDs9?W*vCJf8BT#@)vc^RbhvamANxdJBv9G)VCLv*K-M$wD|as zW=GYoSG~AATUz7jKU)!+1vzE?egx|OS~&pWAy72HXCRTQTLP!U#R_!G4a@1@#fA!PoURo~CF9fS?S)t;>2VdN^Am+@ zJP5*KMB}J~t?>1@O7pPupFG?Wd?4I&4mft|o}#~>AH%I6@CCfae*!Q-;D9Oy3#W9A zJ0A-%zeCK*T5ev;O+CTe7-T&BwYcas+0fV+^AZ0xU`#o+?ses0xUc)o-^joWc~&bb zn}&1dHrQ&)d>7;Ik{G-VhKKemtcY9=S}2K9>%g~J6usq7u~tXt5#GQBtubI6c5TLZ z?LYgNgj%u5uegrZjLp$*1F(Jb!P(aWcr{N}AH)g$C)0bs862UbBz z5g^yQ%N7FN^{KvZV`KAzd-SPJ`2N-&RoxGmZqVbZ!zW48bwiGcSy{I>#x_eKv_csu z$gd88p8WTZ1*JdQz@lslF{i=k^VoDHUBU+nQZTTy5seh+w-s)+5sJm0|M$Hs359ZF zR}TcP^Ops8%F~x%O1_$qAJbDSu*UYGBca!O&iee+cKKHtL0pnPy(}}xsE_R{1o%KN0U7TP=P0c>X!-VPjtNv7> z_G{M=pF99bv9rCuVjwSX(Vgh%I@1u{^zx8}9PCuaoeZfa-n+k z>T1g66}CmCxvI4ebFa%^59T5&X?$o1(bWa`3v#=o-i>ie+fAOi4d)*S_P(Ai zWidzKh6=36QBN^L{HXXaQ-C2d(vcq^X=4dRP%~=si61D zoA*vh46I zX*|7E2jm`eU6>h-zfJ6Oll6X9zy_DFdZGLQ4N>Us{_^~Q^L7z<*;>D!`F!8|N54!d z>R7$6ELSAon3E*!{XC@}UQ7e69=2wkkX`VY@y5R3pz<0Ud8Gkt zi;Z3%J}FGqEUCW9nf&+_9;rfIiAIxuA*l;-6Ml4#+-I!^mrwGikBuhwN8YqS5u}{d zuk=r~?G?Ny!@?Hw4x2S}MDMZP^d?|d_STY8=qrc={WbL(7P@};u%_Ib9OFP8*P6Hf zk$=M5f7TER(NiGPddO=}HwlPxm_Dn7aMU|Bm>K8B>cTA2CwwFbNzFFwDMh=%)a9<* zwKblUokE54N*vJ8EtmZzy?X&)NKeN0Rv zjCRmSxvAd9m@icCzAw^VvcPbq;6K1cv)F8xD1JYHsQf*(vo&EHp2=W=)e%Ly3F}3q zc}-B#REy5NbM0hF*Jb?#TH%TejpXUGfW-c^+|s&tr#+*(4v+stw0{It4%J=BjnxRh zVU>LovXfJG<9@kV)vw*~qL&)jkazM4L6feXl8bjM^E-yk)tSiYPVd4VRE2KgUbx80 zTt0ks#m06hEY&&Z>kFfmox;SH;dFQo+{B}Bi_l`rH<253pl~ehjU0FMaU%AY4>MP&$Y-4J=gpB{X_i)TZ!G)4;dN2zR3vWhZKWk~H(VLxq_(A=Usd$QHWCT({Jb%<>|5%&2suMOE# zDeA^H5{c>#)+DE6m(Kf@4eX#!`cDhx&n)x4;H7>ZeUy5Ur3zcROacxEq22T2faKP? z1D))^UIZe2z_VxRky z3K~w^39xkpeB2qz(}hF^ETL9)F%9!ld6TlfU41jg{jT}e*nJRk%|4K^B<1tFoZ)XQ zSY+s|VP^v@4sy=T>x%EaP832^1D(zMv}Cf*s91Xh`JKQES)F zw73h4YPk7_mB>3uOCCGNjOk5wZN9^PDq|_zRtDp! z&y`LlP&W{jZ#CFNmKKx3OgBECxfbU#-2B5>+Bm_L0YxSEH9bOG1_wDf+CGs!Mi+~H z{P)K(>uDEw_HX1VRbA(n5Ds|P-hB6$2>^DI({kg^hS^xLBeqY1%W3SP{xh47FT%0O z)KTl4ovixxBk73b{wW&L>b|IVR?+|4lL>0w=JXoLbMRl6C&^EilqvcD? z%5~;Pwt;Z$e|FX_VW7-t(Psq|Pn`)Yrh(%DKR!7aniyyf|LE+b3SR0Efa3w?>=#0# zb%a=*OmS8xMlEy{r7JZ(jCz9^N(_$uHm%+&}04_ET# zGU>-7t2|l4SMGy0>g9OMhW1ecOwcjWCpew4bi%GL3N`8W$m#kWkf}#CU42@*R_=?b z9FypD{EMyO%m$nb=rO{Qh&nB##Ic`0ZL8^R$%}0qmFK*zkcIX)G?I)39qxe*)4_>Q z(#*##Mf2Le6QEtk7WajT@ZDjX_#blf*pEQy$ZkM^TN-z+i!_to*<@vhNW z`)4M1*lXMUo(C|e>3d_``k#r-MoCDd)kPTJv=1+Z*pDCf`5UX|&$S)W|H+71b^{8+ zl=9tWP2YYbvOi#I(GB{N9)Fd&^RjFTpmi4uw_S-|r2a5j6!Pb{i99dwmqp}HmSv{! zt_~el)M_k$Vq}0kLo~)~yv-$ZHe_~vJ+Z!VX9lrZYxo0zi;*di?|b|El<3uL86Nmt_8hmiYQQeQ_=4Qntb( zsE5Ao)sd+afJecGm)z7PEPo=(bSnb1Tz5SYM3sb~a2_dgv6cHdyd>LGq(=f63I}RP zFCa@wGI1#zkDM+)p4`rmx_09?IvtGsl9RK+vD{s3JH87lhS<35E2d8w05c)00MQ&@ z7NmNLom?*>v}8DPMd;|~0{0u8!Q8FHH{QLXaT3D{A{rVR+uY=ZZIXjY5jk?=i{-hc zVNl{=;oj?KV+W%~)-WtWVMo;54t+`6MDes}1$r~(P;hhaxNVwH8xg8WsQAyWCDU}F z@CUkg78u5h|IY?ktuH?~mFWNW!K{LP34{d@x&hhe>9m@L`sN9vq<8XIGt(+y`6<1R zhKHoD44Z?Kk~YCw^)x%rTFUPNhU#2fQE{&KuB=$>b-*qDD!WLC5t-^?%0#SoHOHb6 zZ+$qBg|*liVpb;1cjJx+4i>jV(YVat=|PU#K!LFTDd1ts zymVK>?sy278x8o{Mk2|=%Y3b+mOv9L83S;$kH;xsk7#IClGVh^%FEm64T}LjWUpn( zka~gg+hm?Skk-zxObvBcZnoq-wLRFK?8K%xAFCjQ3=9oaU7p_`X9)i*)WF5sYWW!% z^e=0IlHv7(yJ-HImB2Q-DO;|%-J%bjZ+r4fhI3L8|JFNQg~o$bPHboqvGF=B)V)R1 zFva)bx{=j=VfT>P%<+K0^v*kYT>e;f^_xI>7R9}=%#*cUNqYeX9O}shBCG&u_~^(g z(Zk-WFmA@~iFJPSLnE&~y~zda~o&Q6ovcv@KK*~K2p=i(f-X!piO zpoRewq`J?AJ2(r7J**DD8}8H{YJFE>;D+ra{(9!yku>v-x729HJMLYccYI&jJ0VD> zA(s0~qV!nHMQ2TerbnfHn|1y$jMt}qtjZl!F8a^nx7?58gYw7LR+jnGXR@a|%^R^~mf@ zMqMC@RiL5Z+v7q5fnDcO+U-MF-xL||*!Fn6lgxc3g>T9e)r4@?q|Z~L#EA*7F;?4m zSL_?;L!hXDw)hX-nYp+>(33PnVE2-G+x0OXC^C1r3sECI1~kQLRp&FyFKL4{UExeo zl-rAYpmZ$VfCByFMpSPp*}LL*!F)xvB)UVo#WOwi@2{xGWh-y;Tu@rWrYKS>f?jOg4H zP|0wXD5>Z^Zgr`u-Pj?_M%Z5!a3kZ`8`_)J+Q zq8ISN>%&1Lq4qgmz5}pP3IvOL%JO=&FVZ~cm)@kXR6X-gQs3benFa0lY zEar(ogCsAE<@QrYnr@so&N*)G5idsmYAYzdbc=xG{f01)Ng+C~I|r@Sm{J5Vu?CjD zXjR(MI0m4Ev}Xmz?)4Al869n!@u-tOh*fCenKRCt&*fH^1&?GL z9wHeRElBBWhu&S(Vd-1c9w zi6$XgAMv^Z>w5baph5P*@$Ql9p)>r$*Q4kiiJ@3Ua$51#h{$ z7#5?P*F$mn*;27QzCjWaJc8<%m$s2esi|GzVYbZXH$adB9AK0jhJ8|yzTpG`4{x8s zz;lzCqyPWEs}I{MP%0M*yd^F8s7A-pefJ+!{u4C%IjMHS>We>|22X7#YqFx0qwj0 zc*?)pU$-+b+wLF~Klu}|E4Rx%1mFG^W7k6pG5}L`9DDe&DUoz{wb`DO)U4mcMBUwF zqR54f^uCL6bc$wH#7q`CpNna3O48GxN9JV&3R&cCf1Ay}x>3kw@GndyD(b;sg|vS5 z_7;&#!F2s$x!LK?St{zVr2*M%5V@`^5&OlLx2)LZf@tgNz6>pogOb+R8mK}{tfzT` z*{Ye6JDD%ndi()%ouc6c!TsRCZf4q-b_sa&VvpQw5rxFxSub z^(XEsD=Rys8!NA6G*Rxe{hdJml12PMaeTi9xlyvzmHWzpB4d4SelU(7JgB;uQLHPz z2UiY4g8Qp2!xfI?tHL7mHBeg z8??I(bbJAh3-_A-_MpLnDn6p?ZV18u-g?l&22Ai@jedn7pBuHyP z3E>9^7BBcjPa%lxqIo!Klk{^`7EP`DGrbIyt5r$%PoH3AhCfIFHu$U`74+zeg)fOW z55-OJ*5B$Nw;Mp4L+*+^7%ye`x1Pk@^n7W%z>Kfu<^W+JHL6Q+g&G%B-z@Gg~8Ib6`owiKn2B3i1jfbnI|a zayC)5*zC81qeXe!2cxcT-Tf8{9`--tsXxv{SAoX*>`|7$3qeEyHx>Cg3uz@22oVtC%7OcKOZ1QPLn4wy$L z*P{|H*&%RygsJ6B+~ zr{G?l_H#h)iti{wgzik%)qX8!wbK>PgGX?d*)po%L6HMG{QrpK{#lp2Eo6_E(|%)r z<($)bgI+~ye@};j0(=p6X6G~^ZkK5s?|5gECwCK-v@s%x|6>T_EfhM^e^g;0bfUPO zBOw7@c|?`-?8{PMwTA-%>yu|dk88Rx9Rj^ixuoE|d%8Zlb8%*}O%n%N%B1V_O;=t- zC<26kOR!pcmcAUxBs;Pv_g;y3Cu%%*9UT;ZcDTtC5@+q7zqGe~xVPOKgJm+2EtODb z6K){&aeWTPJX(_BpTk|b#fyJm_)2n^!{~gFP;u!!<9(#Q>n|IEGw4-xCl?mxO0p+q zc;>A1Nym>s!Dow8sKDibBzZVh9CtG%9Ee}@W1fpE;C9vc`*W{tINx$>=pOnmaJZ9< z@q6PSvkHrc<+rVmoNF$^n6fI2hBlLr208;s*cRJsRo4YP>Mh>pgk|15P&VDrZqPxE z)TPMiXsO;!=Vm@VJPl;1px1KO$h(e|5dAXDK&V9Q?QEHHvq4`+f7%b`1B=2^nCk7f z6+{%~W#DpxJUr|4D)am(^sio@=49Aow9$T$#H1IJiO#FrxF*mUYbgsQu!vRYbh ztK@5U0JSBRux1A4jHLFgw_0gi-xAp!s;<>RaSo>##;9k_0RhMAUf^R3aA`|quND6t zYZ@KVbFJ5ABS>5WZArQArsRg(-?B99HM;z=Jg}7GYBZAaL1I?A;3DSgj-ukTe$85A ztjiPD!Dd4vqfpm_1FQ;*(UawFe{k~Gd_0c*m4P($KcD#+vL`AgCMhafUH+Ujp3Rx$ z-VYTxP*W)Ynp93Rk(awdi~saRC{Xa&d@0>i^pbgwjON~*udTs!28tJcX}}{EkbnOQ z=8d&_K_qM?Pg-!3Vp1*bYa8LFtvCKey4vfl&7RvbAB8o&@BLzD}$7*)s=M7^BLP#7(_M$A?e zi3&6;f5?7??CM?XubDEZerU`p`w1++jJlWKvm#HUeCJ2L!ISE0hm1;-rOgRJ4nXF>$APnq4Z3K#C&% zA~PV={fDf9VK=ANeOyFW3`Ia4IZ0GUCJPsuV`|%cj1l z1M_$z$wOc-ER}a<;8Yq?da5(!Hb<@^m$;RYYooJB;^lV~2#bVXt|8PTA&su3m$fE{ zXoK#eNKT|>0DW)VhbdggqTTEEAs|EeZ0!OynfKB9$rE}%MkavG{uHfOpQ(uxMo-`B z_t@@zI0m&Edo!kOXb9ToG~u(&l6h|yTNtST1G+4LvVrf(&k*SJcNz-3nb}#c(|S|2 zrr@m+VYM^2It1wH#rs?hA8j6+$FTbIYT8pREcdZt5nJZUwR12S0Q)gQ+;)e>}+sB(7qIR0w6pW^7FMzh>uXIc-E*@+5 z;lsc!oZ-RVl=zovsi`cWS9C(^uKI^BkU_ZA#L+}M$rq8l@R+x%>UNj zTOZ}B=D#q;TJC87@@S+9MgD`j(XG(AeI#`iPfD6BFrj;NB$GQNx_I7r`r4{+^& z>~&-9z2O|?BYilpi$x`qOqod|k)4argc$*du2^A{D<#D1bJJ<7KJD z0_pjgtYD}Bhk&`H*&b2mv*pwx1_vZa}5Wr7Q!xIJ4S$=-4ZmNmpIqTRZ zIP%_<0o)duiUQs6u=k&GeL#Y7%R|;%bP{l|F0%2$Tgb;irem{|n1-rpS%0DHib(LJX zvkyAM{0j@s50(JDM}EnC?*#gt88Vz8R%0-Uq>H0<85R{%XnZ()nUmtoP~a^dJ(R<=Q^4ZK8)izuB#FiT8h(oYAR>T*hOh@!=&z*b@^2 zDEo3xyLoxnsO&kb=>Dn=^OJylW<#(JR0ch?Xl&BQ!FL^ExY*3#WKGu>fJPk@MuC`^ z-+UOv$3+4`5uxV~(&oPxg9F*a11_KPt8PZ6&`S~(Wq!yT=fb*>mY&bMSvgIXz|}n5 zaT^Kp6ybM=yJCLu>JQ5#caybxfXZr6&cNWu%V$8s;utxH3;VyZEd(k{0sPtz1;mr^ zdIcJA#d%S9ctz9$4Vu8Dg3bABHj8?~T_l>=6@~-FK3Bw}0*zyxtMof!3@k86@Pu(- z+T|ONChB#dw1HuxyKMFmP%u1<=t+RYP((I0bei_Xuy9^A`;h$ps|^rlW+aPHbIW=I z{-7YVr+pJoVOZSfo8PsR*#3gv3 zKtFkSIPBn1J>LycuTbiU!g9j?RvanOm0UQ7iS>+sT)DthIo&%yE^Z9ye@7)yb^*KY zDP53>d7{Aeqgi-cFES&l_5ytPylv`QLZf(WdueOIMA<{SsDfIqueywSXrN6_xA!5qO}dl8_XQk}#);|TMX+OOH|Sxcfq|Fb!<(RV^X>us z>!({>7kiwdF`xkodd&Kw5J*R~blO?D{Q%F{?(FC&Qf4k8zO#E$6*2!EDO%Rp_2f}1 z(BO4hQh7LzY6%+04I+6FCk#y8R()yStR|L){m%i==6zimG*%l=`uJYhjq5kZ1L$~~ zOQMirnbc%3MbG>6v{F!TgoqPNAEoNstZr9M4~~2U3M(yUt;J|@HTcV#=(KsE`PBC?`fY$kUoa+wX`BCDiL%~K#8ZUs!C8nuMC{thm*Th)Nk(aGwuZHSCDU# z!o_VDzJtWfa1JuxnM939pim>OJ4tu)fS)ZkdSf}rT;w6kKp9t>UitBl6lkN>tJ zf;nvELQT#mgOa`7&NtZEnOlMs6i7&2DI9fm6q7c>pjlZhy2wYDnruNtAztkDX*wz8 z`_DR(C1QQDWtl-ns(!=yx}}G?q9T5&Q~oDUo^aIQcL6Z#c87_{RF$F;(G}=eBTdW` z!zhQA%%t&}?VX2v64};hNAuRZ<-;^{Uw29N8~0Wu`MkAGpBzTagE>y#3DA zS(357zF;yd5QDtKh`496wySH03!k+F4h;?(UL)8| zw6ad-+qsQXlAQ1Ev^CZ%{d3dL41g-g00R%TYVWq~8cmF$KHwle zr=|inH$?bd^%ce6cT<3BpVQODhpNhM32!{!3Jf*z&DO81M(XU>C-YH}?Q6i^{}^H1 z+ylbvb1;X+?Kw9d&Cnhlyg9W{{r2)<@(pu90+)Ub#+(LS(I}iPrqFQ<&1Um44m%)~ zhuJa##ty2cV?D1a2na_dXm>>F74`WESCB@;E515eU+%57VjGH4NJ%K9-5u&mc2K{S zlig@bhBa~yB()XMrYAc(>YqOG^z%{oMFHm-2+Rn_MF3KB+;p@1JTw<}G;@Ilod>;; z3V(sk_x zl-hi{m2otXDY>3)OwN8E=g}gzM7Dm{h=*Xy83qA>paMsThR$_0B|P zd#MCXg3$MeLmj&{Mjzs>@M!Z)wzcO@w!1=7ai%`#5npfAIC_-H*lJN9_f31w0$0Ct ztmfJJj_74SvV*&5r`>sb+9{=})t2~pfJ9JX<%?#w%%;^rIYGSfo-QRdU>BGE?=Hn*uUXmdyY5mn3rAz$nA?7+kJ^u%HcG&y2JD zcqMAk$z6K6yaF*-Z*%6hT_I{;{b%AD1$5|TjxSk-MMdK$kM0TO;Ybm6(7 zT=>LP>3C$)(U0c-5~`~+{^Yi8UclLJ`-^{7F0D!;dh+Fbd-MJV?=tLAaQ(+@vvifi zRxN|wmgT}FLt9N%`pCgzLZMMRP@o8rPn5)%DCZqFQ zK0RT)tZ%6AqzCs2CH@mefBp$*D4<({_Mb_(N0?P$(jy9Z~b_ z+M-IGwbACcFJiLrhjDmw7G(V4DrQ-%;DGz0yttis8?>dNe0lwylhx6~H>W-h>-~Ih zrf&`*<*fxGni8l7)S!dx2Ib7zM}Ko(a5>6+v=`9D*?+HBolK)he(&9>*(nhrk!m z?~cJ7`OdM|X-)+6F2NT&9$LxaXURps1)Xv^##lf4fy()hR7FFZr|~SDPY=wFfQYHR zo8hj@d9a3v;h3?j?kv%h94gRI*3d}2L<|ldTMD|p9R$-2E7LIWV#_8N`vT(yE*C?g z*nI;7fqo;6%q~}|JQ{fRQ6E%1s>8rohOE!VAkf#To~g7SKO?G=9-S$VnEXis2Kbb~ z#N9Hq-5a#{5NX2Bn_=TrlF?E>5!1Da5WOr};sz$O9;N6p3rAE zR`Mr~@rNDRMHiHRKDiqh>NZ)8xC}l9~mRt9XH9rT?H>g z%F4>XJ5&AH`0-#C5Xetuwniv^Y<(Agz!i^q^;G)F^=BP-*}CJJH+ENZ6i2Un?XmE= zw14|>dNqQcBTfi=ePF%6|NPRPc&NXR;%cb<<0Fj~;+G~e6IcF+K`~o!PV7^0{faa5 z*NT8uL5ulOPk!y+nk1XruPDp&h_7`UAlTu$J&Ws_On zLQB^g#Vo*8DP@`O#$`O356Kc;I3ys&k9c@eu zLE%7+4Y=;+RO^0+->^^k*3MehnWrfZ%ob`p?}5J$$uW5=mnxyWS@6YB3n+i=3ixP` z>U%Qzt)J;v3_p8nidZ`0)5$JdUd^+J;S=ry;o#UL`k!%l6l}5B^^;@1ckQz~`@|nl zb~7JM1-~_U`6B;XU{1~lw`q~F%fl`&FuP*t2xc>ng#c)gnMTC37ar}Na28xl#NJq3 zTyfoe|GwyK`beoDF&z5(^-BC?W=CQ0BN68_OKKW;{FVCu$JSTIRk>|nTL=n@ih{JX zv~Idpx&;KJMK&ef4T^+8r<5QmA|Tx$puncPOKCRU%{#Z}oO{px|6V@n2iVVE&suZM zIp!E+euB}J3@xp>e1-=*3sUOC8BW+M>Vn8DcaQ$gwmTKppgLA(S8KLLie$*Kt?1?{ z`+4&jVrX}dHo+Tvtiow`X{hI(!3OR_xqtx3C|%uK2;XWiU0nNAfLc=#Gm4_JH|_FX z$xrud5ZBWyesEdvGJb}xMEb|ayh9zPb(e730ye?zXZ3gD#=WEcXmF`1)!W4w*Z|?w z>_~LKp_bz)Imh#W(g#dy{6u7@*zF7+)9M&ujE0mukxM`lH=>j6vl%y zDe`-23fNyyVensVCL!l^9P+e1;XIF(f~?dt5zh0B-#^les)#8>xm+huX`tT3{t4jG zFVPQmQ z869**&MQki1V%w^C0J9e{i=U`@k;*Q+{o8E2gF_z^n7_1q=y>BQkQo)iDQ%P^5yp0 z6XK&5C(1aq?qFSh6>*=*h-s%c0Hg}@gIp({w5K2wFfowL7+<<3Ja#cSHL`KF(#XvX zPxkcA!>dAimuxhVcGjE%$j$()LZ?c!>#!}P92S{lg>+hqA1a3v2=U^D6U~kQwJUSv ze-#8|IPaaMRjBnk@aL@R#eH~dy$r-h4aci`UD+p9gr z{W_Zn%lHa%G3>)#t6^C+d99{~&M5Fgs7a?Q=rmPx-F?x)>JyFt=^s+PHZ)kdp=+He zGGO0wkgXY{U-scZJ4s>1Vl}jN-aP>_=Cu%nIIW21k(9Kwjh&rYS27k4LHM1S=15Y| z6|7G+u1?e_YF&u_aZ#Bjs|RWFt_g+k0gnx9x2l1@NY=-^fRccK`|e1bP|MCiPoC@! z*(}MxL1Jp4%7E)P(Y;>N{&x|PEXEjavEg3jj6L6YUgqdEh4)Qcl5s(jUE>f`I|)nb zsS162IL}>Wm`K4o(z|#3q`663;22BAC5{QRQl}X9w{D9J_ox^L#zN+Asa4t?x^*Mu z;1INBhH~65D9>hutS(w+W%Oj=q4H9+24myHtGM>28ZHfwK%gAXH#43c@!ByRucW}> z*m#uvVDuy@EJMs+bY(xb%~5H#9d%Tp^7J#2h0>b{{nc^G5X9wk%IVf~2is_n%`SjX z$olV7T@GJLFb#njO<$kpQPa`+lG2sZa)+>JB*4kESBTno-M6ie1 zQ&v9&aeC_-OZBA^kiJYLXW3n8Y>2v*&C1e39A|jK8pv)~YNx11hJz%k%Rmlo?H(r@ zP37TOO6_POPk)jYMhi&$lU*OC{t)L!bF@#S`8ZowH0OTSA?;z2I%GWdp$Rt%jlE7K ztB?9Wdk|?Rh26*P7&J6PUHlfQ{9guPT9VlM7D1&}iFu3T4B&x$K8G{1plAkmENYxY z(Cvn(TZNOAxE4qE z%r@^Q;C(chf-lc+pJIrpk479b80kv%9RDD8RC1p;d<#^Jz_!+cx^%zur(u0VW}z4x zyYY1LiXK8sgFPe3GYL7R5RY=`{w&FyLbI50YaNG6k-#=Nr^HV=BIo^Yk)SsF&$dbW zAbl`|-4<8;tYYK3R1De1{KMJQU2HXlHA_vw z5N$~9^*0sD&ZX4F9gCN*C_V1S{#URz=!trGK4I|S&-%#T7MSH3-UHdTzy%S5m~b%C_@|s+{w(Wtnh;`4rD{RKmTTc$%cj@7~k?h zD|)syzQ*JP;WhD{RDI?0<#WfPj^-l-4vS<0QMLzSoQyd5f%+Am<<^%Mh6@Z%m(B3! zy$IM-%{5C58+!Q;9rwfNaNb#^jpOs)6uWot-W?~UHa)U=uS-Mb$Q>E1 zTsV)vzSat_s+wD2IILEXlv}vKwO3qdITn%~{q`tLZdG@Af#iH}X+c%4ZagS;>8py1 zZ|yBZ#u%1m*mBYP0Q273b-_(6!Kuu3yV=|E-C#xZH+PNM4B{PcdFi*3-~jOiDIJro zgA?vhn$AbbT~oqheYM3FDs) zyOJumvs)v#$f1M!M6Hx~T z7gIm#G*POrE|{Q0yb0U1W}pA^85yAH9Jg2zuuL1bnqb>BqY5l8-kIH|I69!i zYiMna*NcJP*xFE43z!xfmkr@N7{kqn~rBNYXHhZhR|V&Y5sL``r&M7aZUFswS{viEpb%0aho?TK2Bd0N=| zR}5M-um@O;sQ=L+gQQw>$Q3nI=`K>?Y&aBc-#vNwks!17c6C(?ddI>@;7n`R{2ds3 zm&+e007Pv2QPAbawQKhb8UhGeHE)xUbo;xi8Z|n~AcP9&sr|Z`Pm&H8r2ztgEkSbSIXw!EXxfR^FMusWRvFhho=SuuX55qq$ zCU5ACU4s!n<)q-Hl= zNc9T7tMciFm+4h`%}%QKjt7^cib0m+N^Zui<#rPCbtN2}*x+ja9@jA>QQr1P8}RP$ zyFEJjjv?Dm-MqXUIi~{GJ`#Aj^W#O58ZBE^pO{V)gq(hS>ngN4j7nIt<<6dvaZ_ir zYx!B1XcWKyXvDbVgZfYI<3rlFMMZ(8_f=fuegIcC$1^&kBqxXBm1Ah`*R2{KSBJ#{ ztq;&|-2*aWxMF;QK*`#+hTUOsI2eBDRq;~VxbEiG@ORfgtWFd0y#13hzut^-`F96- z+?9AP=o*(!HS7cn&255g{ozE>Xi*6`KuqHUJR^6?pPijuq<_2w5-yWNaLZ`W;}taC zX*w-%qJPGf=0z6#*p$wVic#q{UH+{hCxe#ij={$2Tq=RrOBE~{$;q7`Kd1VJT|T$MYZ9odD-)Xx`2LNVZQ8kb6}wZ1SzC)rv_bC~grzS@D{MGva>pn{h?e+3 z8`jFXNBoOg_L3`7XV~9t+QoKm$K@e$(_k^*XF;zMZQgh?@I~S)6Or>vmm+5o9V~x#cDE!qmOqm9uJaET!>fK1Nr&%_ zl64EwZziVjNrlb_iH)aM@m<2Xs*v#Hi>3#<>^+gL=Ou+>H-EiSn}qU09bdkSK0Vjc zdiyf6$6Q%v*)(u9iY*cYPWcg;hq&rP<5cL%Sy%2hCv1s}g7PWL#w(RiD@Te<582K$ zbo>r(HN76$x=K;yl~_8)(<-o#HZ^Ww$c)r3dW}3dSUwVW+|!v|lw^Opa>Fsf{1mn4 zk6TRH+a6#2(#QOsS7`jsTXSWJx)Y~M!Ro9^$#b9AM-6MvCuZ0Yp^Q3+RDeD68@xj8 z880iRs|N?1nyag*j8xhP3^c&9mJ;{8WEOse}Zbtu}`qyiW zcBdajhR%=fd`9YSL{(TA8z0X<40#q>GcMLmtaTwTA5mkw&s)_d6Vv%#(vzjbS-Vts z$Z@rw71!4x=5(USbV@KEUo}4*;ZtnEQ9Zv_U{kMMjE4kkfODG=jNZHfiLDb@^i*pU z7>NiG`c+1ea+=&TYW)DiQ|V}Fr9OUD{|MeWG3+)rZqaOGGxJT2E-TtAF5OucS`XQ& zer=gLety)No2_M+8~Y|ja>)CfM60sjv$1^E-J?(Y1|YSJ41YP%ZkEws=lJnja=RJ zy1jk9tMr1CL=)8?<>nlh2dfKOUvH+#dHKKc4v&i+sUmrSrm z<+m1kQ9%keb-tr?fh!DNANpyR>$wgp=|)XmA$i8H~*%BTW4wYr6VPbB|Yvj|b* z^7B1Wu1ZFFPd-ZTw7RyRV7&4jc=>Sgyw0z#9tQ|hxVlEhkxL`0aBqJ}KYF;L@vT6F zU#gw4K#uI}nA_1{VkDCxxs1x!r}vqe?Yt(cQWKgJjM`X+#3@9|JFAbq^oAj+f8>py zZ+J%w7~=>=Mn+v-T?GXN*b3RIS%e-o(hA#)edJ_h{UCT>=uQJOzF%+l0h&ykTAulO zx~z^$(GzQKZ|Qto|4u_v;x;Ks*l~UTO=P176O7>=G-6Kc%3#_HC*^$Ymkw%U@OlMnWjpK=;nD<74-Iqk9k!v zSUFu+gp5Y^Q#6$B0jXY-*`;pgtDk=nb2x`i=-{sirm@-?8IjvGi){FQGxg{%ynfX4 zK9xK^9KmX#B~*5*{I~|$Wjl=atds4;0z=o6UG?6g+fEakZq#+b$6K1Ri6TF%+?eB~ z=ol_UV;}6${d%#*xN&&8-y>^oQ!MCu?dD`j@X<+4q&0U&7P*JJ7uUAShe>;?v&aHH zLEK}3WBw632BejbKSdS9fFZo%XgVQQZvGA0pX!f^n-x|a4`2wRh zFoVTh7{yCo8O^gdH)0H*Kkv>^Oay~67Z;bAWohX$Fxc%r%UFM-(o1Ac2-|+8mGI9i zznNQ+IpY4vY0vji(7n-WSJuUge|_IKT5nK%Z%<@@{su)&ZT)F$0?{e!;oj@rDs}Vw zjb(>c#qn2cb}kFwI(@S|Gj%xcRq;Bxzbs+ombGuqWnpgv)h6!MOF<`_5xH~O8RqwK zGet-|j9*tIYK?jnM=L5E`f!FVBhi;0vI{3x9|s62_|s$@E3TF9BMWwY9T++&gG!}$ zI4ZtMz5MUHb!62HtIZ-&b93G$ZmAs7$Y9dOtCU02h!&(HiR7?*hK1|d%*@Ot6y2#e zC*vcC1+70k6SyQ$vXWLMP`q^A9(w39w0Z?_%{u1hpJ5DVa5>mAm6B=#V@O!Tp(-~g z4i1i$QVtY2LA)3)To}wVD6yKLprAPURge9v^Bq`%!uo}?zHc;^SCh)0K)q$Xdk=V& zH_y=~KDe*}O{#U^Ta@2)WfW!qywy`mDN@h+yyva~z50ejl)%Z1k@k&*n+7lI@3>ry zW-$z+;Fose$r+)Z2X#fncamx5!K&tSsR4!3^72*g`x~{du>s2*%+(W-m6gr*Qrzf} z2M1R;(Ir^8gr#L=W8jwxsY2bYH$Z`vfwow&d$nU*vA$@z7TCL3y13+Vfxb+x_6%bR zr29YM^e-sSr{NvvK7z(^zTviA@<;LG)HwVyaW(dia+Z~YzBm)yve2UFAlGIN-c-Cv z8e21KzNLA-)r7L?qamFk-Du5MZ`^m!0a_pbNH}SAv}|f>s=xu=ztEEb(0GC8@qzL_ z(Z)IwHeQ-+v}sTJ@@N?cE$uYS1+VLSgZS$y6$IBdQy%&J$X7@x{BXNC{S3&MZi8+e zE;?t3{5qGbB0i~SaA%q}JI;tbTGDx;r8!)j%sPFsOe@XyR;}2*?yFZB&+ok%BB^wX za~Mh=om>0FwZwZ_thIt@?Nx-!`fqW2`@)Kf!*uz$bj8G#$w?r35S%~%dbG8!MR9*) zCXUBS`1y}FptuU_x^d%%u-6HTc1#$&xCJN)zM}$3%Mv(TnywWm{ePY;&%#pj;y~x! zHb?0s(Fz5onA67Fly(9qFQ}_d+I{lEFBtyybfsiZHEOD3AeY6N^M&*8nj*0+C!aMJ zrdbqFCmNhTb@$r+Ro(9ycco%rhmBXdaNM~w+4)IynshvZQ3Vh)V^Av<7|8=Jd(h6V zy0x|SH9H&oDp{%X`fnP^ATbGv6HIkq#AFBRV<0P{0+t7YM?`eBO7gMxufR*cXN7(h zOSp~ZDKeaE#=?RY+&_y8r{ebU`Hp!0j-A;bITnZW%31OKIU0E%$FFWw(cTSrw2n!W z+uyh)<(4o!yQJ&_{f{$AlNbsXKSeUws4bS5 zl+6Kg=sG=GOS~R=$9+4RO`~ey%}n zl(ax;t8zM|bCIo9bIAHZm~#!>t_QU=3XGf3R#eZ`9V@eApr-x}PLTfil#cLYZy(QT z+6`0A1;7*rwg&uwO#7gk)!5$No`r>l-(isxh7~a|Fo5~>tx1|Uf)xO>q%JTXpC|N8qTx%GG7^^susc2FY_tOPoy5~gB>m?jpaBe+w z*ZDGNGGOb1)fiL;;5z#1)vJ}26)-p*`KDhFNFjsbM?r9t)$=;;8y_FQ6U~n&PERB~b0#XBR$-s(>+1^( z3o|Ih55oT&0!gF6kRAM0?;AAWk@0>5KRi~g;==rV1Q%EK)4b=9%3xW2a-p!WSO%X` zZos*~kq^#8Jcn^R{4tocK95Gra&N;r)rnjT1xMbr^UCzmTb2VsD|8!y_m|f6i|-@? zqY~q7{a38UVia|@Jx%KgQ}y#BW`E-O;x@73Z2?BMMf!>+5R+-98x?+qlWzVB?={1O zr(7m#}#Yp4MNm)aT7hNy+&N7goLe-j`%dPSXzpCVD6Bpy2Il~LN z=S1Z@=cD<$gps-2pQZjfw%fa7Zo>v>r2z-GU0&kbPiR*K0WFFX_Ben!+J0BbhhZs# z$OVRnhPccJzrh4~@EY$M7>Krbq|K+2EcSL|rX?;Wh6|>+$wsrm)cY7gmyIwwS#GOw z_vxnVl$43!w;T}>0cy)%{>;?iQ3eqrMtg&avru!U*Fge#VF_s>cN)OazkP>d+J zL~I=?KmL(vkWnw1JgB>mg7D4nCk%1v%w10ReX`RNovp0iwZ>pJ+OhR2n`1fqX_PFL z|FQc{{LY{&xt+&7N^+!XD5RTJrV(wJkNft=^h_CtaPQwLsn#A0k{Af ztWA4A>%%_av#^ji%|*xOQjvm>voR70qVv}n^W^z1gZa)+;q>yFrPjY-N9?YQxou3N z0DFTqfy=VSZC7D0^zBU$T4EMu)NXz9;YwiFCj-giJ0kzHFSr=b(}s{T)(_n01VZXG zC1mpJ3kFN4C+}SMsgH)j-v?7%7%TG-7GANJ>qV_~ra;~%Jm;9WK052@KJ`n~Y-h!0 z7`|7-6+1A$thPr{;?4#q@EP=`iueT(k6FRjJAdE=T@i~61;?Eo?zUJBO^ukixVYP- z>ekBk67?$Md_aH=6Nj>{|x0A9H6 zc2+06Hm0TZ^d{EU)<}6Q1z|!uw5`EsmB{llHHI<>=?l#*S$eri$AOPvM(!>6HwE|4 zIfqcgODjvP*4PPT>PO#8*e^la>#?WTs%J%=c_QU|<9b15I79 zudd(?viaYu6}$1f?HnIRnnu#mp$=@=YAZuk=7+^U(2zlf}h<){1t@UikF9rRQ<2A!hO-5EG|kKcc7*QqcLJ*WBTrK=e#g^n)&=E zXnhu6m`19pFh|bWwJ`~T$M5iP?6lQarTWggx>ThU*KAELde+n|%dv6-dbtlUO*bWF zYPiU(#A3v8^7~mHo)Wvcc3&SKF>&#+N*7E;ovT|VBP&bb8Ajy`BQPL=3V3ovPCC*( z9J`X!O+s=hL$QF%>0xNZYNva;SnJb{Vc=|VrPj;uI_TVL&0hZ1X*Y=pWtXt&pAQV(Z1pEGq6|Ehmxjd z5Eaa`k#eluq`y&d{3t}$Ith1wn_{wr-HNK_1}y&G$BvM5bCpgXw}*du7*z}`EE@}! zgieHfT+tF#uy7PvaBpslsU#Z>vJ$j0uU|Lt3e)=_sWVVUq2V;WHeB!DPsU$*kkrBy zeV!SANl)6-Yh|RueU&0(tRiYlrN9^$-$X) zwPu@~D}NV2$fn$hPol1P6ulmkFLNM-I?j(|S|tWmxs|LA6|nBRj8?g!b9A&ER?G+T zQY_%y!71ramsfFfivx4vks*x|ldf;3Jr}vTw^8gb0D;&zUuvnNVrNIdQ9zQO-iHdd z0D}maKcJqe{0ZQCs6yd(qN%BOum>WoEV%VUx}gmA(`)4UpDG!3D@*yeCmGv&qi9_9$<9p~SJR6}EU& zPo!WVswYlx6UFRZ>71)AOyShVu@2T}sIwi3x(8@^q{2Bmh~5~-2TX#7nEbWWx?K47 zJa#^oOOq!_(CKOKUsIaNIt3@7!yl`FQ=&dNVgj26fHm9xT{`s;_=!8R~4qYVH~V5(aK!z z_Pv)QRv+Vd3b4*#476#C0Rw?a29uMia*w~C6|E0&qP`GH1!xqzMt*8}c`SS}9i3%> zE;HNjD1KS&D+rKC;O9d3bJ5i_0Q7A#TI$)8eg!~;!7r*XJSDam-i``Gh2_|i*Kv{j z>i9DGfg=fT5;~-I%pV`k_X4>JqWwThp2^_1uQTK*_ze!4Fae&&skIJkDvE92VCJ3a z6BqNLO6x|)MhIKlSCLPH2^zVbm9S6IJhqq$!^$+ej}fP)mrxt|6{3NCqWE6YB(4R7-JNDwV!0#aT8POHAW9!>ElfhZKJ`q1$j$Hkco%cP zDf{iF-;CD;E^@%G!@LO65wE(vS6*^tf-c=5m29DjM+ z08aVz_g-UuT6mOzyhffolS|)M4IzWEVGZ;^8$JF|df+Cv0Q+FR3zbVRAl{=zb_**y zvb&Ja)L8kpyo&U{n&T*xzp|GnG8SZIFS4+-Q2i6VwNs!g;NHHxvpYsQUvv5>q~#~B ze;r$y?$s84>}Ab>!|lNyo}9!ofXLwJrV3P}q)+fyvC{?dfR!{2KpMNAPmQO`elAbw z=^^tX^iC3m8%`2Ds~%G{p2a{b7!yKF`oh7Zkl6oNJ)@$%+GeT-M#sS`Kl`s&{%j7W zhf8cIaeSC3I&%kR9%1Z_G1m!&BOkx5o7E7d*EcK}gaNMaFtSJr<_3MQl_7qIAH?7xV4vissH(b+(afAZlXT}_m(4V-;a$E=ki?yNDfpK)8m0{Ab0H^GtCG1@ zejatjHah$4nVYvm5Vr71<={ng6{YM~_vQco%KjxGdnwR8uha`KkHA5Gc@7@+p9c#i z1=Gu{|MM#`g0-gIw3a+8+La$z!P^+`zaJ4aDocD>l>^czbD}e6reP?--(M!20h8Jc zH~6hcTyfZTR#Ys$c;@k$2hCt1OuT@}`%)tj5$uUmLw^dj|Ew8(uGA$w7WH(zv2cJd zL?k2&xDb=oraehHI@pd2=N|krs(X2sT~u@^`eot2{umT}AUXW+($nMfRT<)za5V0& zb`Ul_fGeZ){`FLAcMPX7Y8L3@NBKJ?x|AI{yX~cqy4CIws$sN?SonXe zUoh|tN{jvqMeDNSF3980nRfMbQHl0_w&+ULF$R;1wBFv0MOK`DRxieUsM>{Rj1_yg zEWgSZ23Guu3>e?x>l}K&e-i>*(5?5bu!{fHG<;&|EQ!DiRX+2eFZ}dCA@%1E&d&+! z<5J3M>bsw6M6_sb8N&FAh`u;cT@NgnfKi+D1%{{bV&JMfhWUmq4dnJEOA{V%mRZ4Q z2pm@_{#y8^7+&i;9xrWU5d8dB?0QCxXK$3c%*vTwxl(?9nS}dDfIBK1CG@WeHwyIn~zHnsivCC|EGGDfB33uB2@!^MWUwxCV_*XPdK z*uDr^P20}mX^SbF?9J;zl~9Tt$3$GJxK^gWJgd<&*ZzUuZPU+UoTA#>yW$f5sG{?A zsFIrQT=C3Hb-F}!RDa>8bP1>+?YnJi^Q8$g*zB}jtP|1O^IA(XxL}j~pADgO+1BnG z-qmmvK~u1tWwfPMX$UJuN%UV#hnmfh4NLs?z-N?4duT)#^OS2RD+6PJ@!?tA&GIAH z*oJ=YSHeVP`G!HvzPC;#^lHe|2`TN|nv#BzENZBr?YcVe3K`FON4T*MUTIhV_pa6m zasTtt%+)`B7Fl%rtf)G074bSb6-xkK`rijgW-+W3a(8gBUr5YNPgfgA)?NSo6Se&O z=R2-EQwd-*HMETRb3;|c4P<@El8@xW4queZE?dx!c&$2K&Zd<$HHo0_WJ9o2$(op3 z)L$8FZvRO5Ke!2rLwK3rhlTsLq9^hN+~a z4a1ghax?ws8&pj1vO~eX{68-&?XE1bS(e0>)v3V|a#7{Cfn`Tx!J8gxCJ~vKa7iHt zdwV0f1acyWa5H!7In$bIjZ^vQ(q}QxZiRu_)AOFM8a;%KR})=rPw58cr^ug{7pBY1 z-WR}M6*h=&7Wwyry=_`}ModN@KId#T(y%;!|LRrqR~P=h@x=jy`6wuRy4Fr!ak;#h zZxwRUv?}kOXo^vn=Bo}IY;P65_oD}@z1dL8^dnt32A-6P0~8cDe(uZyj(6sz_g{uO zDPq(1f8Gp!`c?AgW&M&Xq=Vk^|AP@|Fml+hz4vnfWGcgMDJm>vVP$O$B;DOwP|;^phDfNyMuBx9f=MGx7|F>i zM}JznyqujG>Ja#bP)#i{SI58%8V7%j1^%34SlAD-z0}NHUvytDyD}*?3sxSQcU+A= zd-e5_R#D!)T(f7-q>?h&QS^q?Y(M1^5m{Q%6x@Naz4s9GDUqyMiUR=LpSimQf^Q!` zKa$(GJ9)Go!ilY{tn|UXgM~rs4D2VWN13HW<*)dorkSy$*ynb3>hcVR&cUESj5PGG zHeYBYVyj$sn`N;;N6{_{fxSbPK1)3!#S9a^6c&0v6YPJb7%j37^#^Jzo>IGKh1naHL}f#%{Vo?sKjK)RKXOP#iCP5J;=^UwX<^k zgWTcs4(cQhh8C2n>KzI6yZWl@+Bp0d-V{T`W5J@OK{IL4*i4=J(X}P5kmWH?P9;2=i^JhM>{|E4vX8C zO^BfNXYwK{{Sf=1*lj>!mcj|89+)c|?bmRagpt+>n})^4@;fdKcBlESs>jC0GOK-k zmizq}x^|2z=@CckH@K(keFffsfesE+O!zgsFyBNB!PZ^}Vn;e_YQR<7-#^~&u1@SO z_PrO$FE2j`rIGX`o2mhmeJMr7U_CvcazpR0%JW$BX{-n%zcT$!FXM@)#E)x260io`Gbdn%sAiV!YW<-OrrwW~0m{n8% zy{Quh4*ggiaR_lt5Sdb}z`K(5QP^PxbsQZWuF~Op(8rx6I-+{V(DtQF%uvsq??t_{ z4+OW+arYB3?bkOlnCW-LFqIGKCKGWtG(MhXDTu^QvaqlKtiY5; zK)=oB5`l?2#1!Cb@mr4aLjT!zpuIha5QYN!6OswL?=^~Z*v@xS4OhVQ(BX}fW4Gb_ z7kyvg&v%W$076D}wkeU@CJG+~_owa=6AQU=F4HAirhYUpuJ-KbItx9&yI^_Bzi@koZ&5J&O#Yvu@6*+g-NfIvI`T`*E(@N0YeU{}|ab$K|d->_Nz zv(kt+>v0`Cg>{!OlfqA*SRy7 zky}<%nE7*(g37bak<8rO+*=;bL-~evmW(|h>vMB+^YS_cJ%n8#w6C5VpwZ~?4!s=A zCbX{#XHc9aIYkK?E_{D=erpG!o(X0-_qaG=0;MWSFn<=Ng%$dSBeqh$c%7a=&oNa8 zT0?0%YHDhyS3+X|;ONc)6w6NY-O;7SAo48bG@0V8_F!1GDUUAJ``b*9S>CI6s48(Y zit?7wn9+-cO&o0L<5oXvo_?v*X9PI6kwG z?A@@PL@My?1bpf+rDc=fxwKZ`dBFWJ4up8Hzk=26+10nR+dMwp#g$F#nCnRNm5+%z zStYv$%|Y#{GCPfnBDAzV>dbf!5_B>V*R*pEp##75`y~(tW=Z%MMG>~sO%T-9qnfNG z3en?LZrW3DytBW4m7>|d;C{4sKUa_@SFa{0Fz^=VtO{nh)njEw(GT~ZU+|y~CDSSi zVuezSF!~$XJ|j@~Tl_Msx13&0!b-^4t5EaYy?@9!*P{^;P|qa@EITpY^`7`(kM6?I za_&>HEvEa&m7JF0!f7mPARSl%7X1}0nHX4*9>oW2FxYo#tYR0wL5=%9_1E^4($dmShyBAtyFeu+r9zup zZ>&m9{bH+*gL1(9jXVX0o(-=oETFh-mD3@fAEh{$05x06ja|30iA&gd?IrhBz|_>U zUSh0clPZ;GRpbJSe6c*(7G9W5Y_4AiT0gRdzl4RV+ImjExC95`(nnt|TDi9PU87 zz_Vx17TOHt=u~QAL4|ks!VDOB{wOdmI!Vx|)OdXn^0=wpNATR<_>*B1tca?F3-$< z-S>B*4Js+&$yGaxnZEZhGY#)oXLID%f*8VT%fvH!Ftt^pRn${ix{cB`A3!B$+Zx+c0B#&tC=Z?a^+ z#iXTnjkI02Ry4gWrNV5>duE#LRaUhB_ql~bHr==ihtoBU_k^pCLiT)AyxUfuXZH!B zC~Z4Rk9MZ;mwsPJ^j~c{(Qx?@UM0M%h&^{lz(3%DU$T?pPm$P~Hs7VQ`MUXMy`RJW zpYB-YFuE~D)zfd8CgAfu@WCb5^_ZpB1$6P17=EYZ9#^G>e@HIXX>eEL76T&% zRPWhHM9n)xcZ89XO|mvUD=S9FF?w^u{DQI{EhA$lymhF^=Uz17U=o;tcpHk zh+=DcIu@1yJ;v7&F+A+(U1?`$-hb9}C=E|vYIkLeSbW@lkDXFUns|dI(Q>*xd~S?@ zf@gIwaC`;`oQ1gTHQ#U#(kh9xt7CmCje}xKB5n~NuXSrxsU;*4B?{mlShbVCCgGB{ zN_!!pjVl6W<*N?iysUPiI0fveT=P4(`%4wZM575xW6QL5x*YRomT-KiR5(7qqZK%+ zOCyi3TShikHMWcFwdBYuo=^iaL%78IBX^*mxt@K}crlh$JElPWTWCyKQc}A3F^7{b z4mZZa_;1AFA|z0M;G=vh2-q7VRyAFU5-}xj-Li7UBizumbkdNXJ42gLlD4PNeTo`*Wbs* z=g}Q-wRm$cTQ@;C6MPode*YF9W@BQsJE_&P$7NvAQqNJo6)7kAc%-}B{3i>QLbppN zPOY-#0OgZhRsWsX{*oo`B^Y&ZXlI$G;h*Du;5n5MwzHUCv~VR-&O`JhTf6Gwf`!R| zM3wzQSd2<)7Si%enQT(Cgi!-uPCA>eR<7J;(3oowiIw3%EJdr-MJHhbv6jk}^e<6& zACd&NwhleSL2^j35AnGVMYPm1jw(=8S^ENCdy9LBwA^re!6Gc}FxYO};VVh_Lm8y* zoKI>*IsX_HNo;IZuuE5LN|zHSB3D;gQ?|n3^wTkeLdj~Eq>8{)W>IZ@VvVCXAgUO` z4F(4N&y9nV77!jzaE!dEoy5Nj5*Q}(ht@J(B7Cik^|Nr36c-m4!i=uy2`*r13{#Mk zA5WtQ4mb?DlEsf=y^vvu(PBOUfz21Uz~nFK8Yn`k#3%&R5S!Q{(j{uyCMmpWc=B<& z$AXr^pL9q^kAe7Ua+vdy`6z98hdAjG{`#P2h}o9cMySALBS@i-To<2zcI@ix*34gh z+WEd?1xWU{L|u~ zrKiQ!lYif1n1%{e{SPSa7$QY|l+z&r*$NM3XGwimP{2V>b@#4RXQV8n=nV=A1!gb= zX|xIs4$fFs$xxVKOSYeDC--a17BJDd4IdWit{w4Q=JQ&SlcIx!rk>2a%ir}l1==OeIQgmW0V<} zAzsudw6`BJwMqZleDL_GXH!Bk?~0IwmY-Z4!(9qqM)a>KMQfVANh*;QWIh8{hSPC3 zi`nx-*bjTlVQF#}EdhK3ELw%nE5sn_{X-6eni(7F(GBP%kzNi(fbsxxDxyvR8u{u*m~|dEaf^bY`dO`ay+3||`UgTJc&~32IKW8wqa8?6KnEV>xSrOE9>}fn zIBZ~UZGtdnI`fu*;eyT;Qf}I1Al`u->OM>zvIFNS$8wkie zH~-E%?s6*6%4%z85VFpzWIaKJ-M6&-9f9a)9m%(A+)Jy>z-}=?NB$!5BIn~qN7}C5 z5_q;c!SJQi)c#{S#%Pil;lNgbLx-Q%x-il)|Jb8J?zvl4xb9`{hShcdZcXCw3R03d zoOcU+8eA{5ngpG3yckMeG_Fj)Z8=S?I-2r(O%93DpBQBu;tP9!BJ_VxAk z0EHvOHKaH+ky4fcEF;Fo#>U0PRaYMX{toEyB9L(HHe;j=VPSQu-R9$!F{J)Jqkk_Q zOhd=B80hRAtoCrTwM7v7A6uanEocAV%iO`fFaOo8G8IC0uqiChB>?I+Vq)Fk%Q*y7yAY<4R4$( z^jNQq@_DKk6Z6~O<%xau>S39e7rA^quM*JpWTY|+pEdcAidXA2z_yZ*{bcs+%VuiB z`K|l3{xp`|Q8KCpM(j~!C&-dh>6rKo0hB0Xz0PB4+r7CHbL$RQ$`uhC{(&ZoV&E|Y zjkhvuyFr=%Z-jl6+A0#WnGrz3^#veY zjR`gm46zINZ-;w(I9VJ$*;);Z$8bQe#O6W|1Cu3igXotI=Mpbb(=7{rN>_(kp1%e`A|~H*@n56(~?NXe4cOpWm7ZMU43er(;3Rl)}38 zIL{KxhqU zV_XrJCT2sSP{0#BIXQv)#yiDeuo94lLE8Ip^+T47clk2Z(zgcopp-N(x~k!(tsS43 zs3#^SR*YS>;8Y6-H8UtJkRQ1be14Trvym3dd#o!X~B_=lSC!f-HWuuk;WBboP zD|Cj&O?hhKnYoVCd%n+uw@s%Z}lLw_2%Co1dOB(1&JB(m54)=w-vY++tC|`nCa2=_z z-StC(;>pk@TpTXe#VjPnt-2!4LNig#UfF1)_jbuI(^{hnsO&%61ZBaK4HA&aM=u^9 zirJQcI)^DHdY!Pw`l?lA;70chX`~Rvq?M;UxoPjp9j-H=>rFy@FY*J$ zDm`o>-~;7DRwG*a_SNgxumAIyYK)@S=IHXISxC5qoKCOQl*s3L7Mj5u)Sgl}#Thmk z%eR2+V(9}57KTi)EJpzvJ^56O-ES(Fodx*=-|>Q9vUcoE|@kuGQcm2-h2c1bjXE#p=B` zj(r8D)dKerlcppVB6f2>e~;zJk2n$Yx)9b>Po}kgYPqzf{ZaYFi*Pl|tVIO`JM3qhx`bts-Nvz72E+j7$Wl^i|hznmB$UA9Y2dq?oMGTsOsX&83u z%FF3pC4-^y#IW4g%v=bm80am-P;(oeF8^gFe#`>+`1k;k5jd+yzzwgfuDbzctG+*e zh1Kwn;yD^EduI2P$8C)025h8HOp*a-j-G8 zV(|0;QDp=b;1D|UqOV726A*?is$U*i;I@I%a7lSPZrB!U%URAc#DaR_y)j3k)zo2h-xU>BuaeNe1^krsb` zX2rm8sB`#BXo)}j$f3%HgLRq&6QihuTw82k6Sb21=%9BpX1I@pOImAJkr~QJt3NYb z7B~i&Yw+^LYWB72-7h^a^|x+c3sYWPgp4L^;&e%wDrUgo;~&jbpXq%fQG@VT;YPOD z!H)sgOA1@Cgt8!CR4HCtq5C>_>0w?~{x6cq!Bh)|EAsLugkcq__eVCvaNawT1GD^T zLx0rWyn6O{))0MbjE2kM2S&ry_{-?3ZvN_rB=nP`6p5YXvRV|)@1!{=iN67j2MZ8>8cb142Z$}(U41vhI~~~5;g~mD7J+cbupd*vRt0IyAK@tGHDaf>Dxx;e z?oXp9W!e~yUc}}4du$!y2t{C zcCvz*@2<9S+ELe8&P%*$M?;aVzXNm;+^!(?qUl$F55cHs)?Npx+rioDGLOf zmxtYku}YT0@2PRamdOMRd%=w_I8oqx|1|4gBdVGh{+qf*T%`*`tT0r*h>7irzD)!j zE}opRPgt(yuHL$A%+cAu8iopvU;4V}==;8tF))jJkof+HOXhY+Tj=@5%@<}vI2Rdy zNYEEr98YL8a{_=mT;`DbB#ufmKN1n>5 zEZb&LEO;9xa7(Q%h&a-mHsi?4dwsKbu*W7P2~&-MR78>*9qOb$NyNzH;imZH|+j zI0GZfZl5*O5x8pN&cnk4AXam5VqdN*C13$vAIGL-ub-yZ2b7&RM)W9s03_7QR5^_e zm*#h_79rUDYB*mr5EEuPb1nQ_5}+2m9Q~nZ#|A4B>T(4$_pL{8U3{>*XyY$owAuaY zAmAOMnex6iSGwC88Y1HQK3^IBG-&604g6LHCyqe@4WP4(HdDitR<9>O%!+Ti5-B8H{wFX@?2x3O}X(azIKCwoWk z8?*ykX-%<+b1md2|4?Jo;Z^W$bPNWDhA6K-EqzRFeuRW-D?T+*OiWB5<8?@2IbKMr zbgAg_`+ZIr{$$f5c%r$X>ZH}hDrFNJo05A+V|Q5>WEJ`aJTkYT&na< zQ#TgaxdsEua{3yKG3&lVUU_GC79vfE%DXp2RWp_KZ$&3EloI31-wM68+{EnG53+3g zuIl0i`JKcu&nfG0xReFYtyfF5=VQE?=leMzixJd5eF1Xmvlss;O!#uFN3dOTc7wv~ zcckdi&uFYki|HPwQ+<)p9t$o!`_K2B&=8;@AGitBhLdJPd(;LZ^XUtVR@Vy|Ztu>? z7diBWhRv{mXho`WGid1bwz>adE-Xl0M#m+;Tcu&)Deg53gX+lXV#i0Uvs048 zHTSSn28N-uabrE+AGg3!8if>U`Z0|lH{Mj8pfZ*vCY?Zj?!{QB%*Fa3#@*oK;60{# z@{EyPxSmnm>@bDLMz`XeV0t`m?@qb&c#}4-njPMc`PkXpDwbbpnx;m!nK3iNV?LIV zmJOn~t$J>A4D9l+x2w$8s8eMw|6fHg8K5keJ^at zx}K!xM&=9P3SeQ2RI~b?dW2Zdf{o&952tGqvT{7yU>0cB&b&`-(+g6mehQTlAJdmOX1UVtXi$0UmzrJ*pfV5`+Y)SQ@&%&vIa=qHAUA(V3+T~7w}YMA zWOhBSwzjT8A>d5TW1->}jntHk63?-=1A2PyFX6FjQmXn*DummW_3+wk{dv-^dC6q9 zgN3oP*p$NE*qFFQdbWJYFjol*SBIxQ+!o!mtRq$xA-3jpcnH#8i%)&mazs#IA&K)y zN4#iS;qv8E>_Qi!PaArQeAs7%F}R_5a>%2^!mkzOC%c+U(0|-3bmq*dH$;s~3N54W zdd$V4)EinryeGf+PA*;OX8U1g?*PxNSbRG|tJEVty3Hb;VXv`lqy>tzwwTYXtTnW2 z3N`&!VXMXiPe+-sB`5SX=(cW*Q{LF7vXuZ{)NZ*H(XGB>TipuVTrS_pgUdR~hM4YL z*Pemu4lE_XaoWu(E@am@4d2&rEwP)NpQ6Hg2)3hlw7+jjAeD%OU}hNYj*;Dl_qa#~ zMOyjYu`bsAtelT>L=PY=5EIsc(SQiISS6!Yik;!tI)tvS5*nk9@K$($eiJHj5i978 zu(e!23q_meoypHBr?;)j(=WG1ryE-X1#;VOC53pa)$+aPk&#k!TkO4{_vZxNv0xl! z(#V?#=2ZXIR{(|Od>EbxNOOTfcaXrSCB>|G3twoCD6kkNoHMfjmG{DF;elz*oBMqg z0BId=n>JJPd90`H%Q6ma{6|%WP`_b`6o+Ryn&p%cMm{XZ{QH7rEe)0@oGqN#OM|ZA zSGx4GR2tV?Ey8(f!jGLuqLMvRhHLhtMn+Cf{}TC=tm=QmI)#Q>!o^=(QXMg)a;JNtZuc|Rmvr}g5rLPp{%QPgbB|r@(P>=-bFwG z2WvT>i?Hm3ikj<>V6Wdf{uq2Hk()cAt@t7&g)d#S31IdS^WR{)K2t0K%)o{@@{ zyk;kdUKqa#$5zrG^pcw5)>F>OM_<%;iQ+aMn4o$ElQ=gct_= z%rG14p|?@im08L>o+Ec9{U%%O7aJ^RST;63osZ57nFFQV)~e(HB~%d3GULFWqI{ey zg3r!vcbTseHwHJv0bS@fOVghROQp@;%)=sS*>+Au{{E^#2>==$O-hL%EQ>%BNB4rI=fTqFl7ol5og+Nf4X0a;kl%U}<2g1E!rC<5Tp zE`FXtF9p(YD*_LL)lH@+ZuCRXi+eg#z1uz|1HvztB;152$Le99J}R4bbYpJ7S(5-g zOIsPCD<51ivDWL{r_H|m8%Ae-ZFVR@_3MLnzp&}CzMRT&=wT^?f+4O(XZ*sjROG4i zXu@>SpPi8nrVnL$tM7TH2S*;_`s=2nPfrq=q_PwlN$`}b)L=r%DT&}w-tfP?^ci=C z-Bg*Q_-h`=I8g5Mrk*=tP!;QO@H}L5w>j!|OYp8^QinFnHiok;%6J)!ZLS_=-6+%0u2Q_{!appLc33sW zf{r(S=tX0FKl)BxyARRted9IcKe^Zu6zNbZLcro_x}MTUD0nFB*k|`V?4b3+JMM+t znVB*bXCdb=M@=q$j?o#i^klTt67ztn-=ehZ2M0&4-kT$Y z1Yj#XZ;Xg1sq~N(9U#(JG`oJ|c>Ln%PAl|xGa65edp8JiMOG5hd-d@gy%0 zeZTMT?;jZ01XQMu#iZI}la`UO2GUV9+`6H!CJD^&=I$z^iR)C*oiG8Tl7H<>=g~~+ z2n?2$-Ued&35y9pb%0sN4fgb(!-4Dcm>aAr=mpHld(WkU!SCqkGDrf4Zlb}_0+3tl z$u5IO*X$7|*5_9md-E?7ubcv$q}U33>A~Z`>gwwJe7>|Ggg*lHcz{L!%kYF*<0Hf> z;RwT)O0V5SPv7v*HX9oo%fbnKBS6P0VrOG(92|^yF)%O)Mhln^*WLvK)?^+?q;5Lk zyu_Sv<#Gjg@9ry;yg6@bWMpJZ5dtJ5(kWIJmX-*7slwTS^78WZbT%GdlggW|Q3f|0 zIf8%00kB1x(~>sds*-91u5>sAn%f`jwZ@tral!6v zv~agXEp--OMmjpWa$YYluRMpr(jW^4WqbR1uk>AD8wx?qUBx15aNJRVtD>L~+jp%D z=ygc^K?d3nIH@_bab0y=(e>Vt<({WnBcEr0KgU(R2AnV-ReppM+6Z0-LiFfYMH0H_xdJY%Y8F`t5ffP;3OFhj9=GGP^n0Vf5Z;=9gF6C5qRU6%(7^0IU8 zsu7%8Qsept1|Dnkca2W%u{)gDW39oU6}Zc0`}2I@1{d}+P9Wm>C2&EXYXGk^Yie7t zk?{ZqoXwv~6zi+A^pdOrX@EWh-v}US* z-ZU`+9xXXc?^RL$eo|E@5UyUPy=qK}&A>CW*Xt}ZIA&n>q!WShfbN^f*?4|%%HWFW z9Z&~jw0GvnMeViCTko!zFmv3wf)-~Rky}9q=`Z&!n@;ZSk91J&(sXjmOyUn8KGapu ze4Lq4o4)7mz2CTBB?f@1uBJkZjzz0N{@ZKz1TqY(;UAcL$mZ*Bm+H)k`9%3X@vwp# zTIhuc!v5tP)Mn@6dhnW}JI}ZRAXd_f80&$OV2V=c3Z9!lj+>AbL8J!5!={8jFv^1$ zHy^cTF*g`=^eklg6k4<2ya5%?Aghl*Gf!@AzY-}*GY#VDj{ya0v!PN?cBwf*)Rht4 zy8e8x_ofIF+Q+CRlpM`}zo+P0Cks3K*wT1M(Khr9=xx)4;^}zN8>JtgKIJsDHZ!B9 zrCkM19uRkop-{F-m3t8aQ_5>+sxuIzOl$8A%_<*^ksT!y27aR7#v2NWgKuD#??^1Z*6AIUWo8n9$Kep(?e+E9m+#kB zfU^9loSd8-_Lxd`8hhxdPZ&QC!m`0(`Q>?lwpR<x1|*3& zKXB5L9hshP&F3h)^(7BD*uK6b#tpT#rZaudf$cdi+|kjYQQ(h5#B~tCeoiLH5maMc z=jcwU1R_3SWE3*92%gzMFoVdhsH9ZmzM|6CuIlc-0h|-)L2Fo;-`qD>dGe?HLMfMV zI9x-6f+wp%>iu&Hz_LPG3Z{r(+D!rIP~nWoR0=Wt=VZEfgs5oY6l(Bv`nCw%JNjB> z9BGxmx6!wH@+g>|E`Dlt2S^Hg(AADtRM@}Ad@wu9{!I$P@Qs2f_PM#1UXb^M3GW~${h-LfWbM? zT<1Xg2fx$|1*1nuzkSUFtG6&6@MiLV0(i%Ky)qhdXZy7vl$rsq_clu_)p!1&Kk^U6 zk73Llkd6;!iSI~~h7GgH+G+S#+0fsD{QUgpmk#1VIGFYttf=JV<@uPAOR`YW({pc` z$vwEOt)t_kL5*A*^Ayw<@@x3}qhg@R^hU0{_eBb>Oy7S-PW}z<=jUG!JOKjhFNfZo z&}o013qSn-=|5WIziGOy8U}G%c=@Pfqjr)(l0HXJs9I!=fw(>c3g*-V^@Bad=8hJDT literal 0 HcmV?d00001 diff --git a/docs/kernel/trace/index.rst b/docs/kernel/trace/index.rst new file mode 100644 index 000000000..23b4c8fa2 --- /dev/null +++ b/docs/kernel/trace/index.rst @@ -0,0 +1,11 @@ +内核跟踪机制 +==================================== + + 内核跟踪机制由很多功能构成， 比如kprobe/uprobe/tracepoint/ftrace等， 以及用于扩展内核可观测性的eBPF，内核当前支持kprobe和eBPF， 本章将介绍这两种机制。 + +.. toctree:: + :maxdepth: 1 + :caption: 目录 + + eBPF + kprobe diff --git a/docs/kernel/trace/kprobe.md b/docs/kernel/trace/kprobe.md new file mode 100644 index 000000000..53bd3aec8 --- /dev/null +++ b/docs/kernel/trace/kprobe.md @@ -0,0 +1,57 @@ +# kprobe + +> 作者: 陈林峰 +> +> Email: chenlinfeng25@outlook.com + +## 概述 + +Linux kprobes调试技术是内核开发者们专门为了便于跟踪内核函数执行状态所设计的一种轻量级内核调试技术。利用kprobes技术，内核开发人员可以在内核的绝大多数指定函数中动态的插入探测点来收集所需的调试状态信息而基本不影响内核原有的执行流程。 + +kprobes技术依赖硬件架构相关的支持，主要包括CPU的异常处理和单步调试机制，前者用于让程序的执行流程陷入到用户注册的回调函数中去，而后者则用于单步执行被探测点指令。需要注意的是，在一些架构上硬件并不支持单步调试机制，这可以通过一些软件模拟的方法解决(比如riscv)。 + + + +## kprobe工作流程 + +xxx + + + +1. 注册kprobe后，注册的每一个kprobe对应一个kprobe结构体，该结构中记录着探测点的位置，以及该探测点本来对应的指令。 +2. 探测点的位置被替换成了一条异常的指令，这样当CPU执行到探测点位置时会陷入到异常态，在x86_64上指令是int3（如果kprobe经过优化后，指令是jmp） +3. 当执行到异常指令时，系统换检查是否是kprobe 安装的异常，如果是，就执行kprobe的pre_handler,然后利用CPU提供的单步调试（single-step）功能，设置好相应的寄存器，将下一条指令设置为插入点处本来的指令，从异常态返回； +4. 再次陷入异常态。上一步骤中设置了single-step相关的寄存器，所以原指令刚一执行，便会再次陷入异常态，此时将single-step清除，并且执行post_handler，然后从异常态安全返回. +5. 当卸载kprobe时，探测点原来的指令会被恢复回去。 + + + +内核目前对x86和riscv64都进行了支持，由于 riscv64 没有单步执行模式，因此我们使用 break 异常来进行模拟，在保存探测点指令时，我们会额外填充一条 break 指令，这样就可以使得在riscv64架构上，在执行完原指令后，会再次触发break陷入异常。 + +## kprobe的接口 + +```rust +pub fn register_kprobe(kprobe_info: KprobeInfo) -> Result; +pub fn unregister_kprobe(kprobe: LockKprobe) -> Result<(), SystemError>; + +impl KprobeBasic { + pub fn call_pre_handler(&self, trap_frame: &dyn ProbeArgs) + pub fn call_post_handler(&self, trap_frame: &dyn ProbeArgs) + pub fn call_fault_handler(&self, trap_frame: &dyn ProbeArgs) + pub fn call_event_callback(&self, trap_frame: &dyn ProbeArgs) + pub fn update_event_callback(&mut self, callback: Box) + pub fn disable(&mut self) + pub fn enable(&mut self) + pub fn is_enabled(&self) -> bool + pub fn symbol(&self) -> Option<&str> +} +``` + +- `call_pre_handler` 在探测点指令被执行前调用用户定义的回调函数 +- `call_post_handler` 在单步执行完探测点指令后调用用户定义的回调函数 +- `call_fault_handler` 在调用前两种回调函数发生失败时调用 +- `call_event_callback` 用于调用eBPF相关的回调函数，通常与`call_post_handler` 一样在单步执行探测点指令会调用 +- `update_event_callback`用于运行过程中更新回调函数 +- `disable` 和 `enable` 用于动态关闭kprobe，在`disable`调用后，kprobe被触发时不执行回调函数 +- `symbol` 返回探测点的函数名称 + diff --git a/docs/kernel/trace/kprobe_flow.png b/docs/kernel/trace/kprobe_flow.png new file mode 100644 index 0000000000000000000000000000000000000000..884ec42f97a8c7e88885bfa9693a59c1840485ec GIT binary patch literal 47130 zcmbTeby$>L+dfLD2na}tC?VY-gGi?!HGq^fh%g{h($XRV64DF|AxejIx1!|G10vGh z-M!b~^SwUr_uI$b``G`Da^Lr=tIzXVK~GfV2=MOUVPIeoD9FpGV_;yrVqjotUdICe zBSa%_gMsk|LqSGL^R>}xGOj0??D0YAM#F`np_IQi*%kDp>kl)qQW9ksrHwomw7NMM}kEsDcD31IR~M_4b+vq+V?|tM?BF&LqBJEeHY3s z#{Yfe#7*3jd2mo`5T$`K{wU5&PXvYsLrao=zIFKvy{Becm5f>i|%fe2N#6*KvZOx}EhkHZD+J#2-X6+=- zm>8e0F%yCZ9q)HTmU_!=Mk*Fke8lPLpU<>>Se|~fJ!04BwbLcQrC*WW@cjAnfo6<% zgz+E{i_Za_t`QB$UL?9t`u5n{SjUauhX=~U* zcQWaRFS@1k^NDuzvS2AET(s`rm<~oAdp?SJWZwa^vi|2?I#Yy$ifoo~qxZ$}P-7Lb zyOfWE8(0((m0%IrjbL&P2V7e5omWLyOs-t+6gDpsva`y=lPKiKOwn($5yI2(iGiM4 zYN<`T#%byOsf?1ldrh|I?qXj8zm>xoA+_)Wo8K|U0+#)=r$;-fcer`5O|UP>|_E zta~*j9de7S>Q!evZ<0nkvhnv9t-PlO;?B#Oeb=?#hlR~MU{b?kYw6Jw`1_iu+qPM< zuyfUYUS72VT1v|HIcgD?)1!qH*&`x})2}@{q@j2Cm$&DT{keiVZ#OYXae$Oqz6*eL z_`8%@qUQTElpHj`){?vOiJl+MAP|V{eo-n;soTCm*ccdJA2hW>Vo!*?Pu41zP+1`v z8A@>lK5qp{|ZsXwr3lb`m>d)s?^>P9w%X z^z9qTJA!{?3vpX4sbmZz+78r0plKa*iLg9$N&3AG58RgRW@c^h$NGE|tz5Cwx zK%yOWjLs|QZIzUNmIO0u@N_%eoZ80xVpQ+$RF)p9x)TM$!@p^c*Ep{LjR-2oxPSBV;OM08bNw+5?>LH};3P3HZodBiXi6LwqJ=4r@1Qqu|Bh{>{3OF5-HN? zn7@7#WO}a%dNcJEcQhy8!Gh!UcQi%N$y@*W%X@>&0*dQ0+?=G~5GZM$P2axGN&1ep z4Mv}Y4aXhDLf#;{Z-3(o99NpG*1&xGcdV+|-jAHb+uIogtxx=z-o?Gt|l{b_L%q~*)U;K#2Xk^0}QS=czcgDX{`N)(OM z51qcwsr^`YRGig?YS0$QWdJsE`jdx z`kG?Toh{*8l++(uh{y+0&wC5uVEpt&;{EfsNmI|?t9C4 z`RRB~kl}2>koGv!nqm$kE6EFS+`L;7Pj@1Lg{mK|nu2$=Cl$=NFsmHS)^k%MV{509 zE9{cH#J#6ZP>u`G!mVe{4hF|@@ET&d_3XV7uADTB6t9G#40xScCrU#p${%4!$_%TBf<+#vsevSR1iFdYkvO^9i4W{Mr~P*nn&eB(^tp`BnMq1uO1GYPx^m|^X6?taj|sUC z^bX=GEg8FyH5jgdjofS53zOGU;I`;Kbeh|)5=dx#Tp`CCr~e9~_dsN&l&#!qFx|vj z(J(r(5%J*K9g{#pn-{X{Wv~$Q(WTGWwQW}|Eup)eM?2&TM&9lHkOeRCkl#~} zvzZ4!e_I_cN7R1$_M~)|Il^jvtbQ(ww$x*9Rq?L(Rv>CSpx5K%2WfdY?O36mY5P?m z_*ef5{;kHdef`IeTYi^Y8NVF;@eQ&x0oG?eq*>PA5AhEYo_N;%qfie@^H{u6*q*KM zyvHs)@}l^=J$_fh!*+4Un7Amirv;uH;A~TC*DP0Z-og6lbPm@WYDoqcNJ#km%ZVfw zV|P%xkY;e0dg220R&wlG^CZ8aca}f-p2*sTPT9 zOW`?-w%Ug8ubhbEGP&}aoW6)m7lm)}Bi1eWSVHLRt;fsY;{|(cxQcE;*Zylw@wg*q z4i4L^!=*TcRKIey@`Rn2M_op28c)VHboO$!3P0;J8=EO8330R=?JcFt3)SYZhz#S$ zN(;$X&pu%Bh>top-|d?ngD%I9qY)lGO^Eu(dW7inFwHY_E#C?(XYz zHHgG9#r>KysJ4n~0=5IHR}bZ!BRM{>-rt%pZ2X;Pp$(BOj``Wo75$V)dp#gWqa`F# zqs+UOX^4y<_;FjnM%@l!^}CtPZ)4%22H$6_j7 zqmEvt;eMacuJc?Juwlvfy`6xHzCfKFw-0{>k9E zE-f5^gM2{w*Ibj`d1%cV(?TZ=lO`$xmp7H2l@ z#wbM~%dp{=+4>-xwqJIeHkL;Oko?*X=%y@uIW70^3Aa~%gjE&VCf zW|Sy=q^QX)5xedNv+{1Ih*qMyO=;{g$kBQVFzsZO-%Oen=>h{SAoTgmt_=M4jB`mBfJg&JmrwCV|XENv|LZ-vp$$+VOu#W6W z@!GevM^@M(%*@wD%9$@#ewLF`F1*`Hvy?3nb`YuO5lZ561bzjN+%}`Y5(CJT^YXy@ z{)in1uxkSFc0IOmiPXss&0$Ep!Ger(TBIWnenUL6$9g((WskcALbNTmwEc8;e!|o! zr$5;}tW@T|z3AKbv-d42vZ*oLUU|IdmGTa{A&+Z;WV{O1DUaVc{QS`4V7WrV%OkFv z`p{Rg28*5P0fzT8k*oE$#a?FHBC3y46;zI<@q41I`r!@x%gE`Y9d`;(H;e@*Srgmy zrNiUB6j6k;JU8dke|zP7(<^Q}AWJ#}iZQ*9B=xMPsmz{5655uMCOMH(G( zNw%X!dRr=XSpOAiyzN%vsb@S%qIO>S89Rjc1$Ip4TLM+7${k@di3{NuZEG{O9lgKu zqmc)9GUXc;pLtB5Ul|YNc(-A`m#c9969*G68;tW+;Uv#=F%>pQev#nHaeR)FxKC$ojBuW zV0ffU4G(%n8Q@v{4bt`3^}G&`YDyo{oPO0<{n|b9hSzgq2dDMXjNS4fwZ)K|%ULY9 zzy8O%CAnuvpyQPa~~SR-)V<471FUn=b7R z+@YlZ6MABhP*WRrk}+@I{m)9`pHxDlkZ$@c5gIkPB)#cp_7*;(X%!^LAUKx4tutS} z>t%^8A#+H!o0+p=x+RInxA=Fj-W9;sg(HQ9+};#K**;%d7h2$f<#7c-K!iv_1E;*` zP`aK%Ujz>#%f$WHOR2DrM|6DdJ33?3ULOV+wqBy>@R`GW-$lc0m$qn9+pd3s(feBx z{@n*U3xr6xU9VT)#(U*NC$@vs-*VUAf?$8j3}JDrd2V$y?A3Mm5Udvnd6PR=|L)NX zUGr^UrW!T}nSynG9vg?I^Je|9{0T>gp&^TYl9bfr##}F1D46OgM(b&smW+A%1q_J=re&2Wa?rzxwz&+NeqZ?-+)_|{^)?l!7&9a1PA@{NhN626ViYGI z7wPnDxHYeam7pb4SzUyX`ZDW5Xhg`F)jN`Y@vC7D&KOAWBV@N!f*C$l7=|2hhvup0 zL?sRFiUgaYx0&!iGY&n1A0g+GwiLX`84QX{1J=XDn={kcOi4W0y8_k?|4g?Nw*poo zQ$kziyBbZUY4=n4&;ooV;L6`Kp4T_`bpMjzH2P4h^0CBQ8I7J70gH|gc+u$?LIVrV z_O26i7&2!3O2}hTjz*tZJ^nM+-#JXd5nfa2&wbGc3B*KRxCm!(y?ba57DJWHN^M86%sh821ts71%O3PfiT)*6R)#j0N{hn$b7=XiExIZd`oA&P>hv%RvVf9N!)--evU zt2f-hv8Sof3~3-{dYnwoe#55_pM!bGwE{|<9_xUmb)-QUs3hctcYq$f?O&spz|K#R zOwAX!)`R=``yl|)9!RH%cuE;C=gtgllqB@QJY zpDJ>Hrr2Hh8b<9sUi=~1L3%=bHz5uW2~7}6UUhsC*DcNKW*Ms(8!U=l5RyiR9{D}i z-*uP3Y$f3I$O#}&=_%uRR;e&+@v5BhQq`JbC|;CybsCV3!OypefR2BKIO0*+njP3S zF)TF)u8mu69-3wi<;U{HQ5isrjVEw8v@>6pJa-^-TTfI=DrqYR6$vJPA=h<-@@x0W z|D%Vuqq+DV+C*{saN8XZyS}t*`gsUn)9V*_tArdylq;{lr(f|55WZ9T4V{;ptN6m$ z!|_rjXbbe_dV^su@b(8^^aZTCUe+s1NO!~7dyu>L;=r1+vH!g$c7Zo`f>G)CsU*Bg zY0cVh@2rsNPrTnkW@#+np%uK7$^-djPjW8}H*b}*6t&W|*gRG_ao&C+CPSk6?tDg! ziZH8r?2A!nQ7O>H?-+k4Z_1@nU7HlDq5VkgsjXh$*>_vA{0XnxQRTT&{dr)G99)gp z;xh@`tpCxqz*lRBOC3Y%MbE;R%TaFrX?1NM?$`h(g5V#q;Vf-A zI~elW?Q=_e+)X9A=zFs6^;u<&G>_Er%g1j8F`kq{nFhlpj&e8RMh2QAzjMbe6~nws z?XiKSc>hFUjGv238#Ma)esjC;h_g8G9 z3{;Lk8E1^J!|T_77jTW=bnBw$iViLMZdEA0-Kf>hNT|iQ&EWKR3*`UYlKT7TYpTWNRB@e3L zoE1UgdXF1>QCTxb*hst_X?cL&%=>L51nIGr#gC3I8be zMCauA&sxlZYjtpS6!@TqWA^aUKh>p5`$Q36mtE~gxcq&IXU-_5Jp8%E&xd1#Yfak7 zq_~H=>m1@@PpKn@FHO=;FrJS2uSxObAel@$5CS&qAQ>~i++|Fh=-9nlL;c&o5KdWL z6E~`T-2TA%!KHM9AVA-}E|3-S3^C<<;>1jX2mm1#1+WlPD_2X7j&`59q;T>9UtP@* ztW-@g*cXqMC=hJWkK%Ch-FOwz;56YA)$8<>E4>^o>Z8w3{q>I@Zz}_mwDZ-C%pQO2vQ+ke(afk*9b+gwShML$CD}a@5V!FZB&_zk5ho?Z?|6SHpk0 z#iSm8A1q{ZCb(KZ85N{Px_eAE6Mfvirm8~oiIAS`vhoN@pT>-h5PKC1`bbMM^!bcJ zf-Kn{k@bzfVpNEe4U5m{TaJ^$WRN)1DU@(UzRuV(WKka^euwoZmT-<}s%t7%Lce_n z{q_WeD2aPFPn0Z$arc*lb$vl-L(5xgz8H9#K)6;d5%Hy6qU^utj|+}h-Ynr75b(}Q z==0|B+lCLQAqlqhE+d*5%E5dRX1QL0*Z&Nn7SVJCOANhO;03yv7&)U4t!J(q;w2aM zm4%eeB$`_ny|7u>x_NFWcqw=R^ztTHlz(5)?*F?Yiln{5-1 zw{p3UYseH%W(K;Y3I2?G6O1b&&v9|sDzR;W9R%g`g(WCm*EKitevH~4sD8htaAuz6 zEiHXC7uIuYHJ-TCnI1Pg*}x&4CP=#$SL)9cNWc_=(?N~@bPK2~6o?mz9=v@~0Q-mn z93PEvsho}2gG=|6d34X;5*M8Tm01ln{C5V#M8*XGgssQNjt{(o4|jhVb1!7C1#n(& z=mX01`_!KpuzZ||uP=a#IXZEpdx_}mp1B*7^2v!5xi4d3ESOV4K7FZKD4mEVr)z(6 zF{X5`Xd&X(;tqMmbxHF2+O_uBhKDF-BISJPg`Of2*bLl{! zC`mjNQxn9|M?6q+5nV^dWkJ?O{aL@zJ^v-rBIrBEuMl5-pv0J$2A=?v4QlNBzYFt} zP7zJJL^{k(LjP5q`gWBK+W3Dy%o)+d$!xAS|EI*x{&IgngM~<9Z-BJjN_*Kk`_<%| z26v(Sk$U|FFOQ1lkeE-xd&`66c}oq{LXX4mEEnM)^ck3(!iN`xG6Z8+qfV4={>9?COn$Hua9kerHBH^-7r_G(Tt)hbT5dEV+=C z-C6%ts&K!d9`D=K58(@2HHDe=AiG@SKvfOy4?Um};fRXm6)+z1EP73AahkI#X#B4N&X2pu zx)=}MGc!F;_tjBl(-xLJ>f#bq1Z?Am;kdVxbS1Db4ZnH0_T)W(9^G&*FT!T)#CUFr zpekWomNvIU_&c}VqBB`FTYm1L^D%h$zOO@rPGu*%7CT2CtJ{;`k|YNGwlG8892kN%+UKrpRS)d=IFlDXV-E!4KgbP}4T3Rlr$VLIXQ=}sFB!exntZku`de|)kIek5 zeGybh-Zkbl$rj-*C{G?3RT|^xUdGu|zO1()&nrPV21yQFC@S!SIaE1)Qy^^>)jCwa z(if5p6UI5nyKZIwry3J;1Q1D(2s58B@!#0K? z)7X#xF~@G>j;H!Jbb)~32A4ril!9f8^|sazKN+hEJZAddJ!_3{$xY{oTMvQHYkMN0 zBQ+`tKI%M69;n?0&hG_sH2J^ASx>AaL!6X#=6$|>>oMCcGoMi1x^s-0 z>w4h+OCk1p96U!8f0EL%c50175W)SWTbbMO**AV{mne%N2D?aWgoCj|IXxHO5c`%v z5?a1qn<7BT{GPLh!#gyB&a9H8190%E?|}=mvuz>4xCjFoUe7STghv54`l!{U%mi|n z^QAfZt@<^hG))ljIZPaoG-KKRADR05H)Q5Psrt-;N}tu5=x|vguC{HLL6C53z4oI6 zMSLMtYvxE|64Hr}{pv%#Y6hB~tZ4edTw-ERqzl+696-$F9cYVyCuZ(4accNXkkQV4GKqHIez7>I=FRNNcVtBNM);2v7eVoyi3C3mr_WCB>$6;1Z4+|^_(6^7D62cN zX_o{PrhV9_Q<1b3ddyR+>`CmN*K{kOPb)!XS-mkBzT~n9iL2(hc&Vh8UG_aM!VlD* zrK@xa#`KwaV!aIOu!CX2t_t4yc2(+9yV;GF-Z%h2YBoiRf+TaK!bV{OzzwkGPUHbs zRP)Qh-1>|2lc1oWvy%fBFDm|*19Ql@VF;**J{B(fqF4U1FI86J;uP=>d{oGVofaiy z(^6AuJU69CP~eRPa-I1ufr(Lo5bE8bq*UN?T^wuHHZS&97ZX z;{4E*L^&5qJ#0zKNqQ(F`Tta{d5D{SU#ukqR&vo^LuwMs<|N;QYsNH9`XhLu_?>+? zvHwO;AGK1Mshjo(^}tc3hoWwG-EyK5$jO;~gk5&CKbL7LtO)DRYvVu9Zh%x|HxZd; zIlR<0Qo?Ok-0S7i@V*;Di(^I3GKCupWn zZ1taPG^PkoOIR|(iMUG2pyOrI6LGR8!>@TkJPOKC%>SJ^z*Da$W_hl(%os==X{a<6_q6LN#2VZ5XFJS! zT$~WBV5xjt4n}a;%zQn?XLYtTM})CfeT3vrwtgGa$^7-887M)406G6$`JwD*g5KWf zj$;q;qD`B57rYf_bckW3$J}*A{rj(9zW@uU!g_c+kk&h5Wzn|PO3kYR8cD_o=hWS< zY&^38grro6?ovFj>GD91qf zrk)|DM^xp>YxUm6M%c?7MIp&rm=iv*wiyJ=WU{t9YceeuWTplpDf%0)QI6*E+BPx5JkV1u5l8(&T&v3~UY0F|`5?oNbL~OiYeiCIXa`tA6P+ z=S4*+&XWA(R8lVP%p2CanIn@ysKmlUA|&s^ATsPw0Tp}4sYfU&qbWtBePcvj2V}L{ zj_YAgPs;8;5x!fX&s-?q_1mWQR00$B=ruXFX28PAkrTj3LgL_T5Fa5SAu;#eY+%~J zG-asArgV_<=U6a=*U5f{DS2XE*~j29G0EfWbF%P<3UjGEI`grQdAS^iKfch#4CLO3 zaPf@h4r#_cGrvdb{g#BS^OLD1VwN>cjsQk%!WP8pnfjEhgPS$9mC3GtP}-Gx%r~YX z=9r3`D_R!OM;hotXRT`dgU*Vokw)j$)hw8C(r0uy;378+sV1K5Jd|8w6b$*Y!@OA} zM9watH9OCG86OA3NaVt4@^wqizXuTXBpvmreSnJ}FQ&DIQIk5q&Sz4BFTyQdTj~H$ zX?S?pv&Hjdf9*M23U7F5=p`4;bJ1TXSJH^}faz6HC*9Ge)$o7LqZQ(FJ{L9ZgYPQ4 zlu$+Ib(Y=Ekm9z5&_8#;r+#L8v0`Ia;o}IZhV_}SRk3n?w>q`{b3tcTnzv+`bO77E z{u6?eXrCrR^R5&YoRs31CHyYQKIS_lHs0PW#WSwPaI+S{cS0-lWdGZ^B{a4pGqXStgRDPwW#ujyQ ze3;gm-#29|IrXZ`RRT^vn4w#L>xe(2fH?5MH|HT^0r%y_UDSzF{(#@C*7^%ibz^5% zBVV#iAwhj+`cr9@gc@8RJxVvZqI(jfUqso?lk1odm%wm{XrN68=O-HzuV+4(1ik0NGx%*e!~c4X)}Hdt&b=lv-&Sy7l% zU8NU4QM0>FUys?Ye7pM)o$rvfLQGN2?b{Y^DAmsX7ss2TBa6d}OpdM=>-721Uw2e} z)Tay&#^xYt)qGZoqr|f4i2DH($OJMmkurj47sm7+j?jKJp0v8z`}q;?k+v4MZb>?@ z-|6A1;zzT~gLy}BzD~$aq^e7YDr{qS8D&J97e^0WA zTyVQ8tng`m!fSJpkWXV;iEwzkX|(p|5BbK8=YZxpd&Po=bEhLs5S}xN(J&<+#r@v+ zN9*J6+ocO_mYIRUUhe(m(UA%z z{o|r?>mdi%`9>d9U7JoF%j5O+JZjVcX6y}sO(?bL&gV2qU?KDmD{eoMZtyZ{5NOMq zv=Zhu-+}PV< z8`a4?EuyQxx)o5{IF$A7U#Krw&%f%OWg*mZm5u$UFk!+-t7RoF)4#RPt7Pf-yITLr z^vD{Vb2o{?7WBl{)En|;jB-Uy)kQs`@?^=pUsoNL-6v-sJ?=pA3QVF~}h?z-_4(K?d?Tb|K|3%^d^Q7`zUli9FFbxb`Ws;8!WggQZ+ zg?LlMCqYoDcs~K^&qck3+n&`ke1-&ErglF+cGrubq*fyJ!oQ3}f6*?&uNJ{`mh`a}fT_8e~${oR>r`CzAdMW&RTdYSM7AJ)t>fcpvGlA@}TH!0GW)Q;9$Q z)MIJZ%3zRx0eT;R=Z23sX-&;2)k0EjkC0qLF*W7`vyt2>aSFu?v)t9S#2g|y*z6#I z2)HGy$DH`&-Tug&L-4wvw^((U$1lXFj(NALg~aJ6intZRh0xKGB4};hr_~?9avuS) zM0H~w9Hk7L#29#h<3hCTV4o3pH_c~oDoOrB>Po%Ku0VU zRT+0t(&y?!Z{)iqn6M~+vOS>2?xc(sguf$p4H*#|YeOMY>8Mb^BUZYi6A)p13l8r&;53y^%<1X~YUEl`NXnovjfpDeZ$&^&R-gl2fwzEjEvm{Id>G_kBr|wUw;A&m`z1aJSlP8o}~&L#dBLQ08wn zdX@(q0&5M-f%W4oB#AmKl1af;Dgy;rc`QD{@CWN@bZ0)ty!soF%}PC+?Fl-a2J`66 z)~s@No>nh-{OIMGi_$Q}k`&H;E^XV91*#=dsxH4{DOyj?uA}1%0XW;U9u!}s=5qwU zCTDc+QhyGklru7>5jk6Mw6zp$`>;|#p~wfu_+nV1|X>Ly$LBizI1z21YzYKYs>uPD>%6mQd6WY>u9CW97Z#sKXc78 z>2zhPZ|4s#HM7jt^6AI^o^KCs)~HhfYHZ6>tL;r8EtG_ZqrCvJ#-Ea{<5!W`v}XdY^p_vt}(pv+@Z zkk_A9K%aNiNR=GrLybR;(6x?#5gU_OeVnq-?o(J|{aw9YI+FNrU0(bx0Mll`nHF-E ziE0_18e}?z9|1;`!X&rh>C<)DD}~agpD4^sQ`u;OA|z;Xi=fVuB+toB5TjRvGLsa> z>MYkD@zVg@tXt|anRSkznB0}=;iNyLS0YZ0IEjFP4`ffcy5!nY+F;8RPQML5u4>`Z)Sk;jZ?1UDCVzAL0?o3{XnhI%Irw!4AG@|{NVeZb1RwZc#veQ!hs+(~~Rn?P>{e z{|E6hJmdShp@BTOgY`eJpd)@kxZCL{o$R?fAq)FkRl@!-}b`cb0+N>+yqrlz?pVbPDf5 zJ0#hUV};u7Ci>KSIHNVfr^9F?Da^bWdrYY{xBa&P$&Avme{l*g1 z=Oq7M+a&#yXo^L1G4%QI@#m5YWn&2_@sD%b1D?$11wuNDTN{r~#SFv9%;VM4VNutL zua*qjr*bzj=P+B~aXrd;LWN}1a4Oz9kgBplZ#IuEPg%7o^%GAqm ze-D=<-?M2lf&z~^^e<`>M;+-?IO-dj;{7EtgX%9#(sUJA0a`*ho4DbzuAx{+`;;zs zv`ER_YXZ&G?QD|c&B0P;mQYQy0zUR1zX;Iby7befk6&(+U$PjlC$VGtz za;yy1zB2*rF10TMmn#!O_^Xj)Z#dXE1 z6V~0NAHa~ET@+osX@Su%;9MG+fEW!cz3~O+u$kBslB7{VbLEfMa#G>$SZEbOfrP-o zn@ewA^u_b%siTo&V-kwhuQYHtM|iKGi=Y+P55Hwn#vd8hbXLvRJ|_zgsD_x1p<`5L@=13%f>vgk+z~Kmvj1PW!@zp@&XaLg9JrzB)EHg0a92;h!m&uoO zlYi3zdKvOgK!V6UOr82-_!7KjRx5H86UjhnQ}d((ldZD|L_M;vY*nkjKPLRiABg8m zaP*Oe&u@bFNURvC`#Tm%$HMPN{>PSSP$5=}%AnQn?5oD~E1~7#SYV)|U_@Y$DI;bv zbh`8dgA#{zQ=H0n(AIkUy?sl%e4|u{gcT|8@_|!!j0^Y-aLs4Z| zY@Hl9{{vJ20@D|-q-J!V6)IDiT!S+VHcvh}+Z}RP{?9}K=Wn``2mwzOV!ZHO<3Lde zo?e*?U{EL?$@H$v4Qb$(1u0lL9|u6{Qs4Yf5M2xlmU&MNTIR-7-99U+ql> z3^Hy0DdI`zCC>N>Ha}F2B{-l zMS)^#Ri}Ob=$HC;JgAIzo zN%`Q#%^rMb%=_4XSbsAgVCLrLpbDf}b!Xy>Y2X*9ZhlHxOfe9jj=LcT>oI}5tt;z% z{+5gxgYYkF4wUEJuHMm}O@7d=n3Xm(lw!unz8HH|TsL!DGm~DR0~^eUQInGt44L$o zgSgcR+1XAZ&}{Vf+6tvpa>{UgT)5E!5x6BPqwyj`70%YXlbH&;iiBfd2U%#m9Ux_4 zP5z7K$isyq2<{(NNDTErlJ!h7(lao?@-;a%Zap{V(#TnzEX2G5rhB6jFg-9|{bjL* zPQi5<9lZ91rXq=tRGqX%wQ`_JdrA@tkCVBgX@3=bA&>M~7Ukt7hU4BgDTx2Xm9?KV+QvdwQ93`NK6C zY`^kZH#;G2*1a5r;O`)oaJKIaZIp_b`8wq03bAz?Izkdzt{4-`O6r(cZ+4t$Pj%j-;fdOa4(q zoxm?iV;+rD(?s2-AqSEbV+86L6hO*?8y1Znj4$WHR0U{T%ox7^+Gk1RXuJ=zo_%ZG zTP;Daw(%UK@kcYntmhh@gkH8hGL*IIpiL;QJ=PS@)-=vB)djsB_oUFwLd@%+(dvI# zs^j|1_3`3fH6BQ_=n&RJI=SN1Ijuyu{s_ANQRE=zXF=*m9N-;SMv{y9aDxr!Z!kcr}nNGw(TY1LSYp^i4I;I zF28B?S3pP$;O*K_y>SV%AWyN332pUlf_bSeJ+!r?LkklIpcVfQ!59oX=izEFJx;iG zQ?;w?k=tbCU8K`P?P7#fwFK@TG!DJXWPj%cB2%Sb?PP=aWF3CPuwP`(#DK+hL>%wp zrfB595tb$F!mQ_j4q={g@d*nbx7@Xx?V$tv^ZKuDLl1SC6=;;wy$zX9QqR*FjnZ|@ zeRXp8_HbN+aq5snePyc|vuKwfd8s&8DE{i&{Zi7LdM)1kyV`8wH{GEMMC&YR7M|y@Jq~zl=J6nX08>H}_ zPzQfb5g>XyoJXAvenw$jMy;1Zx*1w*+8U+keK6*_{MIwLtaEmDHh|#n?|%{Ct_(qp zJ=XY5EPZ9WpE#U21lf8kl4v&E((u(M)m1l10<|dR#D`i@37$X4{YueXR&}b&W8k5a zs#~vFDQG-*R~iC)xQzq_z}{3}Y@A%=8*QD{+aiD^9zUtaEJW!M?=7MdDG;<0Q9AB( zadxUFk;9dk@xov?PbZU4)Z8#^ts_Tf;_FxR#Lxfi+0}lTk4`mKdU~0gN^fsyxoS)T zn9dQm0ES0rx`ryWy>Yz;2Iv_5jbL)lO9}AaTe6QI!=2{U(HQJ$(WQpoO6r~+rcnFE zPmYW^i;|+}t3q>(ZX!E5%z`+ifUEHmcG+1j{YQ@70LC%$UUgOoBSH zo2nUK3W8$4fhB|p!P{?8LKr8td4(4s)vZ5%{Gi`BSL^Uk6i%zuAVC*! zJT5ng^hrPBZC*i*B3G0oJ(@G#)%*Z?gUtG!HM|7&TlY(%N9S4x*Hj0Smry3UEpb!_ zt%tY$KXyA^3xbk@p_D+WS@5XAYN#=My$RBO`l1hYMD$R-0CSK_fZ zbaS^fP{?Te6f}eECYs`74m^Y5rmOEBcIzB+#N%Rm>qza-sEpl7LOUHCh1n^I z@81s@`-pLH-~-p!*{Y15h2!Qo$U>`kI}TE}bfI^PM6JK?i<^>r<;07liW(Nh+eDlw zCf=*6;w0KHf6sBQb-;Fa_QpY+gL5Xu@>b$>is%#zd<@$1&&OIJ#6~UAs{EiycQfw8 zQKL#u$|oK*pi7(IaA>nz=|Oykd7pLx&6j%&Sj3l&xSXh)9dSH`G+}t46@k)u0TZM1 z`yYDRp(*0H+E7N}ig6q+@`iJ2qk)U-7&BQ<#roc|haa(2P}an2ZLwRr#dZ_o&KKzS zxY0~6%R@`VOE^Xh2IypX!?{zqr+4q|-Ac_uFvORL4FMdC+aZSNwWGiZpu0(K!iPVt z;k&ibpE^teg)A|;MD6L!5 zIawdo52!G{TF;_)#2gr8V7KUcT`};EN^g(hrFOw5sM8u}-G1)X7^{yftd)K+I?N*F;>V zuz96r{72Ta&nFq&kH`T`_XJSr9VeYKKY`l@)(uJFme0OIqyRZvrV8{XvtsFFs=as0 zEG)o9J^<}}_hd|v)#dYB+)-)Chd)@JAs&BD=esIf z!_0~L`UW&P%j%e~_blAJ4g^a)IyqVF)#O*uQ0M%s$ONB)+Zq7v)edu+swv*BjRTa2BB0D5s(0*D5a1Ruy-@$0aJzoRV6!2fU&jZk*-UIfhU$TYGF$w>( zs6U{>5+7h8)GuWwzT1ybffy(~53U*)&F zL6MUVkCSwY%ET5i;kly>xY)K`ED(qTK$YrZ{5Qbe0kpzSFAxyeN1yYf!p3vG2hzZ_ z#;|KCR*iwA3qW0C0K%~wsrYb<|3&XJG}^pfWb*yv>**i{xv=*nN@PgJpmeppjfsnt zi&K^}xAoC#z}?ZnwZS@5@Js*vH1Da+`RUPsASbELbVN6q)7oz>Z-+!bnD|lWPX5GN z%a(hd>v}_B$DmJ*d2cqZL-H*f;n0twN-~AM-`lUt@{i#wsLLrQxM>{eFY~{iLLTnb zJhKl)knBC_(!bKffdKqn?VaH_vC}h%P=*+ii7y@Ej(8T$AA=x#`xTU3~AtBrv zG1n1m)t}xTXY5T$O|49GWSr6=&{SbFBEH-EsMd8uJcTD&6LB89a?_$a>GNPt*Ec_$ zQcmGdI?RJvy;9?G$?U!M&^P``^u#WRafE*4M8ebqR zKDzA>TPXAzdu}zy8M!ab#TjP`>;U2gfDd(d=DYOEUTlGDMH=>g>I2S7rfZ2|tqZt( zV|5@$BaYip-W}Y_pb|Whr(G!Sb-cSWwDFxtf=ck!z52VfUc1C>C+`8c7e*!IzPre4 z+7gPswc~IL5MPqOjUM0&S*j`G0IUS)t*oqUSF%Xlep{W}&WiR@BSXLIr! zxV8f2qXe2i-NB$BtOj?l^k+?j=EV##S&=zFuV{QV)f5g$B`8&ix!Kv%Z6r?|kMa3h z<-|ZwMkN`d+YH1l$5MXRimilo%u~M6B9}>P=@Fe9{&9)i#us+&rC%6wBW1XCYh|~j z)jKh1!xcE#^AI1ITK>ukC&>~?L zh8cl1%VYDz{f)g@iHY@18h-h%j{L0^xXlDLzoq%y#hHkWN|&&z3gU#RomGoDGGnnf zU%>T&%dx?o_s2YKlyulfbWz3LeE7JJCnPO{lznQn{7Q1?2?#kDiCG@e ztJ3S!ThKew`_PBf%!G;9)~;3Hk+Yx9@%YfS(xd1n=vU|u=!fWt7zApLm-EmJ?DL+B zQzG3v)d`}tlI>erRt43c`>|#9=5q(ezPBjnyarTI@jEc>KtIt!BR>inpVafz^sXj+ zio)U1`iaREWQ@&@Z6*0#7uLM@pMDN(kxP0he+bjyjK<5;^!Q|P@%bz;Jm>fTC+}&Y z+DiMeazf?t>!#g#yUN;Q)uAU@K}T@i{Qw8oG5unru19j*kR5A3w*C6jor9%nMXox5u=(4uUQVX7~O~J=h;o1T!5+Wj{C|$asG$^5TBMmAnE#0XI5)x92OA68mNJ&aavj|cmNQlxc zAq{sH%+L4Uf8hFy`+Bkaj&tVBnKSb|&+GzvEj2ZDexS?&$m@I#bNv-=e1VqWHq+D7 z$*-GTBK8Ne@b0x0P5adS223pn zTp%qi4cuY7$CU}0I0Uj81b}$0zSe$++k*21)ufjIcj2)f0l)eLT)KYaGw@IC9vp&O z3pKElufQekbbDiMqZLn{kIBw;~ZA{8<3*etyHcEi{rGz9ii=`ES40iE${QW#~^+oI3f>c)pYUYcS+l~~%Dd~OwJ94`Gq$&eSk z5^uV8Yx;+>IO{SMr*OYe((UE1UhQFq3egTjs@^-vJ&$k66sKB8tU90ejk!LKQDH1a zwgzawn+{c1+|25;qq!){@=RNBRu+}cuJ|!|I52*_L@)`Pn)YHqp(Jhzr_i`I$|^OY zfrB|C?N7wxH5iqrOv=p@UXYx}mYMX*ieFiK(l*eBn*W%S-FTqlpwljk(h$2~T?7(6 zLSfrtZ-LhJ*uF+NW=uMjia|EoiR1XY9!Wu_UY5R0(d9ilxwu59ZV=WJmFx7(F`e)5 z5!|v=gQ;h__!QjFUJVSFN@L1A24~xYwcodjS}=J?jq6ZbzYEoVhG$$SJHKlBDM=lI z)92@jQ(Yn`^io8vm^c+mS8f;3X{K5+dXI9DJ zgy3=`QfGQn+J+_nJs;)H{O*>V=+2T=SL{JgubmS&cuAP==vR)YP_QU9DTQ)Ztjp4e zidT3klr)rs#{g%rK1q81y=K(qJNM#WsvL9WM-{L;E4*`4$>@HylvFb?dS-TLx_9~g z^%%|wi^CxQp}VB9ae6E2VrF!< zmDrQGTZJu>OdsPiPjy(cI@vi%r8nf9t!<<|WJsw>W5yb$(@z@vf_!kxU#9wX@H9Aenrrl~##i4So$AAlhKm{EtfSCSZx zXvu4t&SobL+07T3YCTZ>qEVD?eH4^aA`1I84eq-eQ+Jy-u zH}8#!S?cWVKg1EuJsh@LYRdw0C=PuUh4-Cei~^@b+hP?ewq%r211fF5YtCWRCB}x+sKm}(LrirYAG5uR5}=P%*Y2?H7CO~&n5C?QAlx`j>b9+HVJjq(5GMpLYN^u zW>po(csSF>b45ne=Fb3F6+-G`_!Nz)UFvwOZE?)8izSQT0CoT)wLA`yB{`?ji+C=s zX2kARB;Hm#prGcvw-53+3M2l@C>qu!*9s$iYQdO*XQHnm$P#XiVGVQ51=+iZG5L;)9Or+opvbG-cB`LyJI^r&ahU#E^^JZE?XScGYTtwwW{Wt1Q+p%-9yhh zq6QHxN>8@(WMg?PD%Mj<4?GM7=y^ycao#%<1`iqBGZuUYc@4)3~uyl^u#Ywq+HR%e=g@k)JIzZIK4 z0fE%4>&?>^i?ISwoK#y-{!uociP*%9mbO_fIN7v^Q#`&cN3c5DSa~2E2V;ADlh)(d zGWD6GFR&_EBKC|R1Qs$(7r8!HHm|j|Q;@dM+tTEbbH~nO!~QzQz+h@ zs5uX6CGn3smeshN+1hwJc=$Y*X^C}>><%$%GGHe12obu<1vo&V=fobz5bKp9o2yfj zImII&Ha1>LDZ~<>UB8#SYI*D&?d~;W@4q!^|D5?P$D$((A@hb?eT+}nTTR0|u9)v1 zn3QP-rNwA?I9Rz(U``_7JPn7TVJTLUUk7$>%<3_9lxoZv3RuOZDNVrc9WkEvaaLrq!#s(6G@Phc7KDu0dWIWN^_lC=_rf&DP`gSaA0+uXHSp4YagkN~IpsKa*PsdChI zkq_k$K<?9Z^`1WR@@>NLdr$Syg;Y5_Hub4Xq2m?$gb1}WO4nZ;Ie62^Uh*6*@o9;R2L`U=un>#y+IcN2Z+=DqD z=RZJ*E!@9pp^rCNSgqWR9h(Z16HK2YBEW+>$w@v8O9z%Vee*RG^+PE2H)ywr>^YtUmZ=0vNlj!32@$+tf}IA0BBtf3ptOq9VX3kCctD zC0uz90r|y1YLV{T(U}~ZZJST^@=UV zb~2JsT;KBru$Gt&tgDUHwUt;N7fNhfe!k1-S$7ge&o{eNgK%0RV^io?$iD0M7{9bY zI$@KL&Hb#R5yM*oK7|}QlPL>dq}OX?(2hRKX}6CvJUFKl)t6Aq<~>e{I{{=IYx%|4 zTxrgMZ$w6YQ zs*-bje;hVP+dJ_QqQJ=Vw`fV-pD8tsBAo2pEhx`_r+Zh8P5xdA?(Ti-!sX#=_&zFm zj^r-?$kneH%ICN0KF`TNeR!`5fwaYlUKjy1f$NDpY3C)W2qv+iH{ev|SMjb;siib2 z7th)k;+#9iqJ-s*1_6%Tj&Yih3Ua{e<9-{5wr>*L_aF2#0m9RvaKidH3(WPtN`a8rglJx>eAd!0#_wu7Pq{kqoXtD&fUG< znYufk*Jt&O)%`fhk)wU2!D(adgacOWHt3)M=r|AzAjl%K{piV@4zP3*9mnX8lY>5#BTEqNVwhVO z@&``7EijLA0+zC_6LthM&>T94waLANm172+NbU?=tpcImWA_cw(jnP?wkgM}5Ujv8 z)U`NPR6fp|@n!Yo%EpEUelT7hMro0WzxC14%G_*Kk+nT##$JS{N{bdV#@1aYxB}HEiD*8soJwf5nHaT5Pjdl_iBw?8 zzgP2wGY}$H$GcOjz#Ln8Oon6Ua&}H?_<=2Zd zrG3@^H13<|Esodw*Md`B0)K9Pe*TGhvb&t%=3uiANH(R3RZoN$gPrcOT+-o}uiSAy zvz@^dgx6`Y(lU-C>>|!2Y{{OEmmZH&{semhk4Mvv3D*HHx&4cB2nxnk+Ni!#Q*dlM z@DlruIZ%JczN6nEOaa6IgspjLJ~<^&8~>iFQFg#y(p<}aDB#kj&Vvp$36sT~v%@n^ z-?1~82tq?PRIpJs|V|zn;05l3w=QNqYCL(ElNMx>kv`hDKCr%N59u&XtT)ZFoBFtdA!T z9FjhcGR9)tp8UQ+EHY1hYOucgPO4)PclVqR6xVXD8_#rHV}1Oj`LuQi=|z9ydr0){ zAPT3ixT&0F_>hgn^B69xOU7o={dtMH&!olN?kKX*wL{;VKtWzt>ijwcW0z;o-UT%p zEcgmR@M7;>fVw4mYt0C)474@(B342aBkxy|^-}3V`-Aq6Z-yH|VqW9jlrgrGk z?5|e>A+B%0_tF%s9JP!XV$zb9N9(7hz@26{mU)El0~RlA;i#qTP8^q7KP)-NVAApE zwfR74^YAm97_h|DM-!+LrzL~IPvD89T#lU!G)Hse5@2;Bz=mOdLpkvHhQEQIURUPM z8UzEWxS*(Jkk>u0(u{A?ldfHYv_xtJk)=IEZaj8ZPN^9IO%m4AsPU!X$Yup}{mayQ zC)bN)Fo9|sX0b0tGOc46 zCi7&6LyR?h1XLFI1n!Fgy@_k`)2Und3tDhU#K4Z+Aki~u$v@q$GOC3Z&|!!gG4G7lat-N8h? zRqvTMk-s{TRkE7F%UgjoGi?>?KGv_pJFnoqV_bj8L~bZ89T1nFfJZn+$<4|Nv5G8b z7q;}tc7%>t6jP3nJ5+oSWh@A_caVFs9v#^_uG5EWluZ}qu1VKWb#Pa(Z;{6NRNYBO zEk=~YV7(8`HbI@_II*j|GE2UA+9lYDktOHeXr0qy*-Nr+#erV42;1>`B2kQfc0t$y zb10udd5{PW`w9|7eD3&!Id!9p0=&-WLo<*Ry$v!XO93SlIQy@V!kv_>bgXu;wmMXU zxwZQFE%Di;q@=VHCqa!5^2=?+jzD0u=DBho+u=drq8idF}wG>)2G>&KpJSO@f1xuaNjiulae3XQ{MD&g9bhZOy z_g)}1WC<{f^8C11&=&Sykc-!X8yzox@B2=u(9B8RmVxxhREmFSeKq2C;Q6|tN+9e` zA?}{L_y9dFvT*q!Y^~VE?A0}l7o7X8KC@2`(8xkZMS;sx0tf&PA3^08D!it>@Q#-ONxKMrFf1~~d zP)+(ie#Ee8uW;OoD$#4ekJEtijzZ+Yu!qR=IC~aVb?Wi4Q(xQsN)J4Lbeiwh8)>iO zAns7xt*R@dlnE{Yo&}Ozz<`3KN{safSZTI4CqmxDQII zLOdqIM?oy-5T#MMyFI_VlH-w%d3Y6~WvFC>W&T3!c_CY)1a1qxeTw7AX*w7>ILoC` zku~RcXsj*1E#DN9w=nT1-ywEaJu(H3wLJWBbpDK5N-iMQ!3eN8tlnHbQq3{m{_StL zuG$8hLE2pc3RNMOZadn`Jil^N`&htDHM%RHhWFP`9wD8jLr@ao?7DFVs$dpfN2|^? z>aC^Pg#a=E8^zs4*MnFzc&I(p5^D0QIm;5HU{ibTpuCcH;^9?*jNo8H#sUXW5pOvF z!2kr|KNS*zLjat+7UG%y5*v8-V}QSb$~-k7JD=GJg8IRhDqjjH9U^!WK7SaU^#S$6 z+Gbh6eYXy9(g_M&-`4K~k;EyJ>_3<(AXPA{W{=>WrKZkdJ8ZBo8NLJvQ%;aQ3@Z={ zQVxyjkZJkM`^Z-5LEX_ms4#OD4nDqU`cGIW!9PD_0L?=zn{`#3>ne{^RW@14nx+4vUGuk;UUrXc)7FZHV=_@$(=WcAQSj`pAYV(rOvsmaT z;J@|^hohhvBsoC(EIYux0SjXmeD!l=L2wc%KM(+a<3WXlB(NPm_UGLQdP!wvK`$tp zrXk?erL*&X5#_O}=M#I%Iy-oO zgw?)vAOJhz6tpq}qT0t!?hBHG13HPzHmU=t*Lw5l($y|=Z3fEw@2g2*Dko)*>A?R{yl^)LtJ#O6l*PA1JFrm}e8suO5w2IV7Ldzby5Ke`t8HgfC}b@oQ(#)#;V z>sSom-pU?(Iiw&+mpcxshFCiK3HuiLD9ZS*RV-e4b$#s~`g=~fmnFX17|A0SjM=iz zgE#|9`tn!OvZ8kngV^@|PR9*q!mJxioinBAS~uc5a9zciA1i!yQ}A_5C=T|wOq><; z(h-Hux8W0p7l`ndL#;1-JKf1yHooJsT;k8*4yvl4mIk9LLuc#}GvjgTTiKLD`271m z;(X512a#`7GF3;l*S}mky-cyT`FQ_#tecgg{jD@2Vrx}1p+mokmEJbFh^3i~-hx7ip z1kzTn`I@<(+31=^m!N0M3U-Ht`Vp4354#4)^G^JjX>HReSfqC7DT@v=e%hdsG(VKM zo7^ozcI-^#RT<-xEF7K(F#&9zc`YL?`V}36=ND7l6KLMSwHx*W9JO%VfU%^&u3dvr zWS%Y)X59U}jdsLz#(?~Lq=P|Uf__Q1LS`3xt>o*LoKCLQb@5zLq*I@w{_!P}T(+Zd8AQ_D48pS@;bGZZOtmBX%B zYhW;pZ?s-H2$`45`1VQCbNzT{1H9&2=-w@EVBXpsOa82gj~JMId`ySQ0*x}5p}^XN z8fIp@mE1LVl8cex@7L1{bH?;VI=M!}>mgv$%7x}`GurNI(bc_spfo!5hEkoKD@B-9 z$@cg$(Gund#^@?jgc#bh(+l;hxYLm zLuyvmPIVgAYsxosr6W&kX_$K>s}L`r-etA<}J2jB3oB zKr-#wK%un%=N(7#Se3x29Tl*F;Uj-+;1aP*b6=$0{m30ICH#A6(b!{lbam4Gc4nz@ z@7cne0pwtuYC601lrA752|r*kE{+4$|F-x7N2Dy{zT-7bkE}rl6%$O zd3&;li3ZYTzt%m{&OYXq+f#(Wga^m0=Mb)_PoZjMoO1>h-%=6APC<)-2aJY@B@E+G z5B$tqy*R87BI2{Um`^cEfK-unj;gR3M>P(+ zYVv=Xngh?T?PY-8LVYTm8cf#B_I>up{h?0Kjw6>|it{{6+wCm~2oU+KQREVUKTUpE zw?}D3K1E-yS^X6!n0qI}7Sr_@(HQINk5tSUaGgX&8Ns2dBo4uG_c4rayOuW#d@+ za$d}E$seDAHJuwnF?0Wr)KL_V*j0AmOj?{&)dM&--?qeB%3zz zUa_mZWKoJ+7oQI;IH7~71{Dra8?BID-J7X5Epu*VOX?LXH%jBq1NzkYw`GsnO!ZsU z9cBFbrmbt&s}%d7rT4S@-^yo|1vD2)pWbt(y@*uy=dG&RE%%>#Fz286QJbT=IKm;f zpdAq9(1ms5#yrL}kIm5q&5H#{ZJoLMCC&ctnb#hfKZ;GTZqiST&(`k$ntQfKr6dS1 z>3)xyoT4?IVfJiyWlyZ#+{mGoTh<@^`{D`Z_|$j_)GBZxAh`9nDvP!g!?_G4-q-idCJKXOhxz6E1t ztQlCD(;4;%Mo!J0+V=TTxm&&IwiEJLc`Pwv_ro*^ZDq$MlBnW|%y~rxExv08?(g_R z0tGvXGP3Q;8qx|Pha4w=L>0CB%TSv$332QMlk8q7OYTl5Feh=#cG1<2*GG4)fvx1X z+RyaeB1---F;-^ixUBy<{w3qhwDOuJ_uso5XJPjiw2kS`jx|tw@Pq^pH z^N@=Dwip@P?rWyK(lEhQAIE~P=k&ZD+|ZtO!gczTX6(K`z0@2>EVLsg&tfy2cQnZ; zDFK%pwHPWu?+_e)ExYejcT}OP{4#rKiC!!1=GM~p#lv5eH(SK071tBfmbW#<4;nX> zC+{w{f_sXaT2AU`3Yndce?N-8=`e11D)vhF?Q3|Lk$Nt3``tcDQ%AqXkA*I(T&EL2 zMDnNm>>+5`ZX`oKXoa?DvFU*UXHxJ8&*zw}o6xZAxFp|DWZ`sQ>Jfo~jPrsFwmiK$dn925= zl~7n0M(WJp>ZNOot(Af-87?hPqE1)Ay3(Yob?+v1c|4QKnP-Yfw-Mc@JG}usZz%iw zQrt{K0wpK5?$hVpD|xA`zq$ROa+m+wO`SR5Y3~O=V}*5NBAIqFXKt2CFqK$>pfTD; z@O_Nb)O!1v6-d-g@rbnyaCVfChCM>wg0vv2*I& z;N5li3%v-58`FcTk)DDKz2XB5Jo+tnk?S#XV;>f&EPlK;JAJ(EJ7z=N zpm(;Ax=y+Mw*ToA*9&=B1G!__r49owyKgrR$8HbJ-|nq{e?FC^=)|wcV=3xeRr)Ik zs%>>oUZi-k$>61j#^SyeM7dF9(F@-$oV(xA?#UG3lrmNLT*yoa|-%T;F$( zUCXCA($tLP$dtbIYIz=gGVO8$E7(#7wCCq^wp~J0FyB8Z1^b(s|Mt%1XG>g1JEms1 z2hI;Pyy{A@4?C+4jhkpF;CMv>kIQGnvHs2_x95apDB^BRx}7TPRsGvV(sjGs-0Ani zolJC#kU7p!-V(_U;;GuP8j9(j9X*1 zi?5EB-sEk7ZH9Gd?%PS0e43v(JL8eDO<%fjp3~$7y;aM$tvQ5&?k=A`^LuX2IXf%` z2tz`D8R>r+D#$})s7(adoWz3-OQrbTqs5m}gS0}odOsf4YzjcSC7n+n1f*aNcfRrn z95sKQZ^}QGIIh17>hCHJFn)RGZ32Y zv*jtUDn|}FYK%q*=JczNxB6Wr9vQ`lR4NquAm4Kc%gLknJ^Yv*BS|NX3D7O_%6U6X z?BnNs0#&!uCxJ>#Om@YT$K4TzJnSEmNps#4q-qN%+*qT3lk-vML@m^wh*djCWy{RRnhQX!E zB-&L0%OQUmj1w+0Ma4+$yVt|$-%-X7ZM-InI@9lPe+VVOF*5$xh$!Jg#eZ@!J6ZjTuHzzw0v$7^+n*}-j*!4k*!_Z50z zY|@@-ICRt0*toB@!SNma<3w}MtTbu} zJ(%f{GtyGX-f6zX4a4;1S8 zVj8giU}Y%J%6uSxXDhR9=^Y(wRHM_m@A3x8-g7yGr?cN}+fK9nUR-;bOj&~C$8=E< z0aBRD(XTdkGq&`DMlXI-CXKk6Il-pVPIwbT!DnLzuWaDKFQwC{*OoaTFH;dg9rAyj zWJrlMJwjdfxML7(ZVc;uAeo7m#t3bdDM%OXn$a1yHI_hOtR@G!gT{G@VDbO!s6#dP z%q6k(8HO4G&{W<;OKDVGC z17brwy@{|>m+(y_oW3Jpu!JSbY+gs=lb!PknIs`4ENG$Foq3)w7}pCi)fQVb#9&>= ziNu(GMG(>#jM@{ux3)=G02yrJ_aYCH7gDcL5(xoE^1fy8hdap}{}rj@9!c^Wt*rHY zcv6KGUm2*iHn`OY%3Ft;v?6>EInb8r=K<@HbmIVeqOrUbauBpfFfCSGL`JnE!=>UE z!E68Ybw>~hc7}@tkcHy^ZZr^PZLh(~?V#D|#ry@YY}7u=Sk!{S8o=CCByhL5M)* zbpPm0JW)K4{qsfa1*9=a&q;SfoJWTfTAuxRP*m~E@2|LGk|u--DqV-+DIHt^c*hd=*uSOW1>L2`ZaQtt=N&{ z!~s)_ z*!^b2iujR3q}cW!P-Ml8X)ohw-)HoJw{Ybqh=8Pz1!VoHp@*h#N337aUYX=6IVOqU zw;2)~k$agi;v=GS*zrNbRlMKRy+CGUKEk^p4)r|p{pLl;}tET)5UGc6O0mV?6;dYYA-%MK7k>O!y=>JfvgTRK7`)Hpiw|fO3DSeKTrURX0Ze~ zwTa-VH!aZA7StE1`0S-+J~A*dC8~%^0Bflk<+QOp+11r0_Vld6;devNUi^765O|`O zqi~%_NYQIJnQ9gu?@%aPWlcZ_>V@1;%8cW+%Wzqp3+7++{m@)cPyp;5Hy}Taa%fbc zB!s$;FfT4H-U>MPeJOr7<_;mM>4TvL(b$tCFrxeO%n126pOaJsAMgi8(kHlITl6I1 zz8olcbp16{b6xV@OiF|VhOhV~(qx2-#@K4L*=GT;o|`ZQd6tBP_VQ&;^Zw^M(n27y zne><@ViQMCPmi$>$YfaLtTzGRJ^)-2_$m|xU0<334Pghs!+dsAoF-iiAv{2Mg6YUN zcuyoX^NcnB#fvXc_KUEH^?-883UJ3hW|2QQx-B{TN-?<^C}%wGTaIdUzTA6}2p)|& z`XGn0Y_SETnIU;{i2hwci}wlA&@{W5Pgey6-D`abSv3lby3#^kOG}mlU5#G)`Hk$7 z2cNGi;E0fkGYhSKIjpFHlJyj9bYftnr%wf370@_)p-Xs-zF$YA1oUVbi8|^&aaf(M zgf117o3U*gfn0ad!yD=83!W2_tyLX}brjb;&6A`+VI7)h@s&w!EOKWqJE^uO-bIsO zK}`_MOWaxmmQ=;UyA$i{>oHN!o>>Cx3J0^iP91Vt=}aaDs;L{mWQ4AwpOuu9nyyWX z0Qx58kxDS$`Zwy`VuSbRd7ALIgPf$LrGaJz%DKYAE!0UfLSXGA`=ypcP{epWjX9&y zL{^)luOK|rpGjWU^CVnldvj%{uFV76$vTL7^X7N-HyV+h@7py#Z>2Dkj@OK>cj>K` z*kgA6!|-IO&U1f}ku9xMsQI*>U&*6cmBnOpH@?GNpg7$$4E6T=MrPN;X!DF6`&~h0 zM5>~;vfk3z={s2(-lpmVd_Lq=Y*ZYEysYV}MfhCo#wzcU=UaWp6cm)9w4eU1CtKaX z5MA)LpnK!GH3>l0UA7CUx-+&Hlg-;=8CyEJhBNl8DhL^RW@q@chi`B&X*y`OPJc*v zY=&+P7N_F2e8|57Of*=)8z2-)LQUJPS!!!+W@ZMw0Sh1vVYfGC!M0;Q2~t#@7h37It4<*T81gG`J!&jRX3XLW{Iale#r>N zCBvtrsPG2{+s>=c9=n%MT*BK}3HB&=Qynvq6i~5TYohOa#vQ6lSL?CW(mBfr;*p%C z14<9((`~qLnY)GU6avcwwT{ll_c+_>?yTR(OL(mchsP>EOO`PGAQq*$ha->pfZ~89 z^a023L%I^Z&&6RUCnul{U8YuCWoroLMI!^0H+dU*q)Ux{6n~ECK>I0~5 z8K4T8W5d{VxMe!(>28RhCnCdZH5>5O8ZW&M9-SLiHYb|Tl>0gsR+6oq9}t$Na=8NdJO=&6 z-*rz@HWJAs3dm=Ej5i`L@lWM@^AhE&ECO<&F}|;maiFW>Fh4lQnxB?t)R80+E>o$U zeMPVSB$v`%jXS0KO`%NgEEbWZkgcrSw~MN3gipADFFH>!#4c5CUdMVq1!+hx6+Pf) z^5aTLbeN!OmWG&)HO=k~FnWCQ4Yu@d;s=_mF4LzjI-S{EoqGjlUqW|6*XM_vfw>&< z>IbnSAhT~);-Z9BvQURT+)eGpi`%!I=1+}pE+8WbW~`! zKfi!PzZ;gCmHbpz(N3JZ|DhF>*2lly5s?Vq$=Oxw0%mJ)!!~0sxQId$h^lHdTKzJhD-^iTRi* z+P}jp_sng4WSCxvd^0W1cbX#2EeRGOjAN%vcl>l8dTS;+|7Z*=14jA$3zm!C0@g;S zNd&F77%CtMHI2uBYD6$8b{4(L(VeHN11TUua+1dU6{Z&KhDZZ~c8plwI*@~FC*om; z1v=V5GN`#&QmM9fo|B20VCO`3#oF^FB~lWEiKSFZ=lWS}RfgG-c_ zKDjH&tTD*tgY@*@he6$xM=^Fi@b#!qyyWx4c1b9*39;N-Rm6~Afuca6)~9uco>d^< zs=ae)X$T^lvutoRZ#kJfa$>wPio+#}fr$pyWVbTkv$?EV5@VZkC8v{P_OM3ECf}IW zi%=zYdAU+3Mjny%GDcs1y4+I&OH1G4vM0z>597pA{Lb?2JjJ(oHrVA0;fTyb z3R?i)9%SW!pzi|IbO7marno?Q=amb&CLLNS-$3S5Jd`v5x@rsV+KEK&a_!PuICral zxQDAp^vA5Zbeq-iCcPdy)@XjKlSpEv3xSjzF}7Ficg(mr$a>Ys9=Sn>zal}ET+5p+ zQ?*=&zi>v(F17#19;W-;?UD zX@1PM&4a#^>|0b?6TlKU_(h#~ZqmLpc)GD_;Bm3&|3 zBlDD|P>k$1wTQ8%rS%jwc|eoy-s%^F8*9}{Y)4SI@Ag5CkCKs3h(RmXhhpARw|C7p z1P@bw3((0i3(t`{oxg*9PhU-g*9r+J#!`*hD@z8;*RXU3D9`+f>mL=dE_FB<;>l48 zOva-+5dN~%ojtm;4el?s zN06zbs@xusp>QS=_Hvd~**h#7p@VIW+DnzD*#>lODBJRHyBhX=KJ)7YmGY1!tOADs zWhFPT;Tks~fkQaOsNx5n25@oX7Gzwu>r&L_jzZ(g*Ao->S@cGUCOb3usphwKF(ZNJagYO;rix8iH4m8uNaam#VUG|4Q!f&81+MTy z=2agY!tc5s?m6g`2;Vgn+C(RVE(2XG&uRXJ1GJ!ECMt6{!n^KK?qNte{X0$%IYZ=* z{oUFy4M1CCa6J7pD;K`ZNefcDj{fXN2)PjL|M|+LQV!@b2z>GDyBV9wrzz3LAeaCF z9G*YQg@YmQXY^F zo0uC-L6;Z2%^z!p97h<1cdhTTILpPbZ`*V)r(>gPo)8o?u@R}VBshyn0ckc_=0V`4 zx!DhF;wxRkm`n5ZL=LD)v+VmE#;PCi+c~g6r2o7TK5D)9t9KG(UhOm7CQ3as9)IL@ z(~n-Uw00S*QDsACrh3W@*IqlEkfkCbnXEQw>gLf*L0X2L>3*t=Q`ZID+a9qG z<_=b&pE&Hd!^`o2D4}PV2q+t3Zpg9yS+szbMj54uR}g(bJe~yw+7`n&bD{fdcL^!` zgwzX7BF0PY1-DhQ2gL~s*tpslFiCW|qEl_i#d;>ndK!sb>D;04t>&AI?e^)5rb*af z1XjQwn*i0@(GT4+MbelL*TnHb)S$V1c>qn;NgryIEvuCg`QkW9;wnDx?W5``EKg~X zg$w5TR$ZnssRF2~*k)A6E9yhp!b8J^^X*>$ojeeywts6b?AI7`f4uGN{ShLKfk33< z%+7t&EDev2Nak(VgY8^5*>1B7eZ9!_Ns)p_U!Yq>(QC{1P|hgxXP1K4aAH}kU#zxH z;{?*ke=Gw6#P_cZjKwrH!h1MQ2v>H`@JQBwj`a82x*i^ z@r>3P9g+_Sb~pr@FT**`&Z^*qBC~cFRMZqIpIdN%^8Fb*+B;d2{f{!?Z<4KB?$iA)a0~pSy*sk z+dEbLvwmO-Xs^FDbad`WV8N%qcXFjKgNT)i8v`x1!ahN~ zTyE%#(P4_`d+I7R-rVuh&KP~adZosR)GE-A8HH1=g71ZcEV1#(zrjwwm)H9DF)iT6 zc#Y0UYcamQ%g85Y!YH@Q+0O#Gsb-~;Wudby9E#P^O`5ElDY13U0u39Fo?*HoHIEtM z?^;R!47mPsHgM9{^3nH|F^b#WvH0pwJ76~WV4`0k)zJ3^GVgZZ0P(nT*%OsCq<{20 z+2`xw;w}%bqeIo2BMhcdj7PMqWuh--Kc|HS|IhAv6so!o;Vj|&i2A(a*&|acNb9%u zSxk&WgqgSN)amBgO7yjB>)xA(z)E`i0EH=GP-kEkz6v~%Dqw#6%E=o)wgv>ME0y z6SMp)zy3S=z?)zZ`y>WayW|uaUU29C@vTE#we|IjPS#(l4qiKT4tSqhS{ih?1c^kj zx&gZaXRjmX(9@7Ho*ra9P0n9|mfR%(&ljbIO3<$o0?^-h1?3c#l)5?ZLU%qJU=BdR z_-Ap#QsmaHWAXGrAW-gncn}r_#MBZF1gf-Mmk4pResUA&ENeilYn|Wk=~JcO->NP= z^b1Z!Ts*w?{hMnX$h)N(S7DnuAHpmj`z!d)H*8yULKt^k{`nHv5G%+D(FOUJchKe>oAG)iwKF@k6WAKIv%Q&#J8YFO;`+lcpu5*9SF;`(246ws zj(-?sKc)M<`BTy$Aj8~IJf=$t-RpI?ySi^U{m-L>#&6FU+QuzUw2R2#kc48yKBr9B z1gIUuHCl|6YISvyWIehDllh-VFfW~?4Fto6?z zXjy|EUc1xc2a(h1xY@nv$dt|djf9a`XN8-b3WJOwuKCaPjoTBC#?r&7CMW7k5Eki_ z(s=dWgGwD+3$pAUz-0;^Ho>IPJB$`1fLApyQMYHf=UZ4M&T@bx%N^J(+ z-fm9vJBAJEu|b~=r#_w&er7Z!5Y^5fO!q&Ng|M4a7F2xi&r z4TViS$-Q9w?ZH1Q3-Y0~t{F4HM3w0*c3k)T>hZm7Y%(gXHt5o4Kn{gu7!4)68JMTU z-Siuvuar;WlH9TO>kFTN&34rqW35}W z)?#PY>$1%!T(GU^s}R3lLQa0_KqC5uED7>}BS8DL3B;b=-`UPDsazkhh!X4%FXMnV zmfnka%@Z*lIR88}aX9Zoqx$Xi#DKj{S@JKKF7W9qSKG}GSU8r}u9uYNO1$kp zCU)@O`Cf(BrdUmFu0PC-oXvMhzTBp13nYj7mUFS%uYE|L+(pCM9G!ZS<0$q({>;Ba zZIy$6q1nnKBWaF(^-i1|$M}?qIMMo4v_M(`K;Y(E+lVlb@xv?O#Vfq2fZY}T_>zXV zybl=~w$t<0KbSyIr_`&Uqe4NlV%rnM7i1P0>2nvYPH*4T9Vk7M{{MVwd&WdbCvJ^dQ0o&mWstbE_u}&0BUqZ;#%8*kIp>h zf9PK?JlB0tlxI#Z0*l$6efbHkJNKqo#0#>j3iT&Tvca>PJWj;!V}DF2*v0OEf~SiH0*!L_{; z^yDea;d5BRrm*~Rf>gdvV`JaW~2rD4)nOwxz6W)ODoG{^rQHty&SI{S&34(nDe4jH9Xg1;9%D zW&QLNU@2nHQzcCIs- z`mVeiuTg%*ObJxDmNE&{@?HuFR9EgE+bt!>duZC#$)LRGkAHcTWyi9{rBvSkT;Wt9 z03Z3RmDqJEUSywc#(9T0cmVz-`oJ0rj_WO&{xtU9=T7RJZpHVi<_EVw&I^B{ZUDS z8EwbiA3PF_N`bT`$^PKHxDT*p`u#CfW&vgvB;nz=5B+KLP)?u^oR~->*^I$_Gb`C? zV3UjE$Eq1qyRY4BHCxr9Q>{M1u3Mp{rsfFvh>|pmiuGGex4iwgLFuuU>4;m3uu%zM z%eA{4`wri?AXrADK3LloB3k2xq!9uH4>8ddGgGe zGtaL1Z^6;e>za^|E^FQCrkJm{=#uhOgw~ zJ=3|6;TMi{2l+|itCNRuM~#K7RjC!!)EMqqy^&_~5qdVRpdfjgcAM$WJ%(WMC9fv zS^qtw^8kGlsktI#Zxf%eKWa}qcItyv+tPa<9XWJ`UMR1up}C=IUatsvgeZm*HPmpKxDmPe1Vme>+BYMU_<*%AGv_iDVq?OFXrhaoryzLLP}cj`Y` zI)a<(%ATSiFv-b7Y<#!(3<^Xc!G@VbxzIet!booK-wz`K52KfnNOz{vR>_p~{3`gX zQ?SVvj>Tf);**yhBh60pu)FV9KI-2SOZgnG>39gPVnoPwRHhWd5R;q4We|Yvu0%7j z!$IPGSeQ58-bEh(4l^AcT}#1jjd}36K7-Ow`O)UVVSbU0Qo_d_q{DCOb3T6n(U+jX zbi}{a*Z<#n8kd8?nrA5oBtAm=1&P#D8)9<7yM&UT>3PWNu|dI|EDdJ$ikTg;-(zo~ zHVS|B&n&h)Vl)(Pue8Nz3<<6qlM>lhii#Hyj5-GNKV95 zEs-lu4~nCQX0T(-`~DtN3S0MKFeemC&BCV8CFWA1ab!Ga3y9L`+P9&}olHRI=d zt*JF9*Bcb$PJ5*5rposr&tSx)73%G=C(*NA|IR@C-77_bVp2mA;VR3qIV$fyyyy1d zO@YD=uSsDqa9l+tzhRyYqLHdSR)iS|Tt@CwZSA%_ZfBFFZXSOhm~TxLL?J4pm^gaXu z#E)n8|L=-x%Tx|9p04lc>ME8hv#v#o{0=~#nm1y0AykU%<&GS0dmW=P-jWDBY2)d} z0t)O>ZTUUyv;4@fnYY_hHYI9<`X{9^*E%8}SrU7pmI0SZJGF;%E0m!OpBJM5%)cWY=`YzB?hT1r}JhTO9Pb#1? z24pxxKt^@5I^j%bPyR&|5~=AvCClJ3nHJW+k47b79qi4vf+SzvjpbPEFlyI5T_?Yn zKm$|dkmQdfe_%WSm9G<=0En)Ig`oJOtcOT6Zuu})1j5D*EcE_Ts{oKL@sfCI7q-@y zrWY#GFJ*z8?}S(VbEl8#fFE&7S8iC!7nfzHOvADuRx5^y@D%?e7PMdd=}#z4psjt~^XmK78azA25b@MT~>oa@UWo_klbzR6OU= zO`VM_QDNbSU^-B8ZtYhSE4eyV;lgGt3)38-pnzZYBfVP6wekr$4=rzh!=bGm0n87wvx$466IGR2Z z`4yPkW~>MkK-jvL0&^`Qy|gHmH$#+1|4YKhbXk;q8+ZLT-b`ki2zRh0rLlj@l^a4N z0XdoV0EgIBPFIkJdgcsQu`>i%no5^^bVEm#Fp|We`E9T?!(_%XQwkIBCN6h(z6(j^ zTY{JL?5B{RV9Xbg5C;3xsmr?cyL9K=epSgVzH7kUxYjI@F5{|+;Ad@^dpYcKf?WE& z?}E!SmID~=Rg6x-jHzcqQ?7Nj=U-6gniFTB^{T zz^V}R3d8vqSmlYlKu1phJ zO!@DL6kla?JR$DoF=I>lC_m=6F!~6qdlA6Zc?B_hE&D+N)3}PVwhqxb18lQICzxg2 z5<#_h#&w1sVB$xu>t1u6yFdh|&};OSP@k9t?*+3>i*3V9g=D*E1{fe0tn@=1!1mc- zJ94Bcetq^u7Y7GNJJBNAog~$!?b5iu#Y*+~RG!@llU*FU;ITKLiS4!DD`G(Eljz-NQjJqWKBhA5Wy7M}1Bh^=IEnHz>@`i?HOr-}l1JbVX~w~I;P zt$Sl8!Y+-*E6>;P@$p5b{W_@{a?5Xi#B#1rG#@~;@}q&*nk^@fQxC7Fk)sc0tHfjC zot?v}n`31*buVW>?c4*+*Qu*og8FJ`CU|8^e_k0BtK)YOH>7o+AjC!xC{(-DHfI4G zUtkx$c+alVeaI6SQ0iO1A~P*R@;Ny;!2VzZ{2PigCuGJIwX-e*Awo?hmZeZ_SHY{b zgKZj2OMRxY&PAWN?Z{eD1;e{ZtH74GDF_+LN7tm#RC2 z!55#*b6TDoPBUKs3pf{B&3HYAXt|rPdTxCuzTLHSX9ll_o=vf!Qvy9t+RKHuU2M(E z)oW*4N)<>2+xV=kEH=fpr#OT($iTq!VA;LQ_BFc7P5Yt;7C2d-gqlTdzT(I zw5VtYLWKj+hGycY_6H(*W!w3N>7~ITGDv$zX8~|Xa5ib3+7MUwnb0MU^+Lt>gRF>* z*Jub>o8&>mJJ?kSxSL6#_JctA@;(E`C@>tf2V-rf&+EWSZ!QBV{#C02BG_fwjNkRHOtLNzG zIC7(fdweVR-R05)tU)}zf=>qO@@r~7&8uhcWnhtUAG|LY0#Wa%Sp0UK&X3lz`*QhrXzIJ~mi z(OF&3zGPi~l{Pf3k5f8S4;8aOnq)1mqGvs2ed*G9a!N`Hzkt9%snyU_^%)9{_{A$l zU1MXria`(%nua}o^yN)-D{klR+mmKT*>#(HHQo42b)TS|$SgiYVCW8zqrw0DLdF}8 zR@r#&@l$H&E0S?;eJ{J&Mo(rp7AL5Ys$YyjZ4`8G&t)lFiJ3;~71GX~%X+NbOrN;8 zb$oGwh452c+-HwC=)`s@t+k6+K}Bum{^)?nDh4YoVfE6`MVRWWx%u;;`tt2gD<$vv zhNk~Mce$aDRPIAOy&#{X!isptN))AlT@t&^l+i*e|vhF%WNlD3iE!_ba+(ah79vs_CdTu4>1zM{Z zk5_bzV*Q2)W%bEW*MMq}8v zOKO)o)!G)WT(q4_O-@94GAvk;8|sYUvhA4X^0oDLuh+?iJ8Uosn}<8Qoqc4R8`&-W$_;y5-D>l=d5;}4S>;m?ZT&vp zToh*~Ve<-b_U~aY*mlP>e(Xy;gDWc9`DxKOK5C~_u?r-to4X@ul>V!Uetgl0mM{;V zKx7Al?cfM>A&)i5qdsK4f_$NH&r8Et+65)}?!32emUqC`)x(k-uK3NIdSq9D(>l1v4&DZh{?4(tXy3QV`&Emgal2p zMylK`vJ)nz_f>Yx&dx&2$fFi=W!bu75mdY?YpQZ``06`MJmkh^ug^H{6il!2G3d-& z5GTtfZOCC;aBnRT>R|@CXXYQGngEq5Y?!>t&qJm^$kly219~q&L+xsDb(xjMlyNs2 z__`3h?fx)Ud-JJIR|?wl4)ZDoj?@<_O9rg~m9hiu^vTd?pqx~UO1}&zIiG+)^vIXD zc{RS3w|@T|QMCltB{(5Gn6g9*s7uMN?8TIn=5nr*oK60T5}S70!EVAOGycdq)O~g z^EmO5V@h$n{lACAyH2S%h`1ZPJcl@CufCzdbOaK^K`TD`I`oS#;Z^!RK!%8l&b%D< zTypKgfZY)4qlju+8*zGi`f`=MIPLQn!iy(`*c)Kk0j90rLM}1Os!+vWKSSF7-M+9r z8ZuUCDh7UgQCH)5q#W;la;?vI{0`8xSr0HF5o*HQtOHe=zZ=Q~VC2-#<6H3hOPQT| z(0U0a^6h>l`@mafN;MmQIg>n&s{X+LRZ__A+cNU@F!6Ig^Tz5mxFnT$y^QVU|1TKx zE;ro3lO?~Z&Cwd!J^}9&=;lfpTstgTW3{aK*|LZvIq9IC>yEdZ16BF=Z)(f8!&G4{ z?`2i_x+I!X_zn&1VQ4!d+BG(Kn^V5S;S6m+clFKrrR3!UAf702sjOQ|yy=KQmDH;A zKN1_U2s?)d%i>$hF0kEVEX?IHwcDpyYQ4W8APdNe28DWCR~ILV?&h#?QCWo_>ZM`m zau@ime5v*_p?@mILv~nZ&DQ0wnfSO+zr^y$Cq+y1EWPn(xaR*StYyJliWprtzEcI4 zw`IKm@cFDY^Ybtb+4;*DA@~nKx+))&N0@x^EHxHE2b4a`8JYark|K! z=?%)?c9Bken@DC>mT@pPU4OH^S1Rbk?`%OGN>iW7C}1@CLnxFKip2D{Yg4}-6gJ8E zu-+P=pz@FhT)X;?r;cZz$vfrI^=+RfQ3^Ko0SEka=ORP;_HhDo&4 zjXKY>n3B~kYxmhbs$q$hMm=Qh+*s!8+O6PMOG`@`GOq7lohMjol>CC`xXGOW`cdNR zg`Qri+O95Vs@Ca+gY(uCdeh_^rpZ+u;8M5Nz9}Bp?_Ty=kn0UNU8q-X0EF`3v2zid zTnGfJ#fK`)_??q_>W9MheDdD(ku<}Lgf8TK|IsJ+sL>MvdA+q-oAEp3wNa{$sMNLf zkxOE@;ybW!TZDwDun$9HpQKSy2YAW}+1`MC{GZQDi&)CWNc1e~d6j*uIu*Ly13o;h zc1>#3Iv(eahrDC+e&rp=#<5SYTQWz)rKM4wZLf4V-kI}W5UF(=5Ft-0xJn)gaa**6FiHG{8stSgT0|qS{;FxR!82eq~*?4AKO!K`}S$ z`V3_TH%+^#O7YaMOTlj7DFhv9c_HBkZ1~!sp^;0`hd+F^5AA5P#iq-Rz89ajC+NDu zHou~vyXgmM$$0JU7uSDwKVFd!Hz3VpadufyKLB$jbNZJI7=qdN2fPJl$FlN=HFO*KB~rMaQ9?TAET)6HX!>0@dWm{)W9lJky4W-q78qO0KruOMveuWSW3 zc{xp;-T1v@Dvu|gR|F)vV5i5wLwphe`p(d1%7~ukkGHy@7@hXrgcEx; z?ri2U1`_{RdwaW5*wz|B>OCoTLP(izOOuZ`$jkt;k`VLi9Rvb*y{>YWMY&2^Rn=y~ z;!hN`6C^C3_#vt3nD00gkoZ|=#u}-xM@)3ij*N^L?eNM%KUl2#_A+Yp34X!bL3gqw zXyitxv7QG9*T!w8x zUSOl4X@%gShAFT^>IfU#7490}lzoLYH3}f--!z{S96bDB8#1vWix|qJ9|IJ*hH*>< zHz=wBw5wgJzxz@B^dv)hsoLt)!g%L9W|It;#88!r!ZjB&TZZmo z4LnAh{9X*Gq$@f0N;ItHJyeSL)*=(S6L2pLrLJD-di^>0N8zT#b0QU>uPTUT=+|P0 z01Zt!TuHpS{N@_|ta9wJSMQTH6b;$LaTyuIP!mB1bmwU3Gi1EL{PaCP3K|kgau|QOQ?e}A;fy5se@cMRxJtD)-jawx_y~N$%gaqPc!4BckJ+6 z)L<-X56znp$m*==RGfBiz9OLSBI3zzb&w`v>i_%ZYe)T4{kIvz53D8dWHfr(##+xb Huip6|(pA=V literal 0 HcmV?d00001 diff --git a/kernel/Cargo.toml b/kernel/Cargo.toml index 18781dbd3..ebe69b99b 100644 --- a/kernel/Cargo.toml +++ b/kernel/Cargo.toml @@ -24,6 +24,8 @@ kvm = [] fatfs = [] fatfs-secure = ["fatfs"] +# kprobe +kprobe_test = [] # 运行时依赖项 [dependencies] @@ -58,8 +60,12 @@ wait_queue_macros = { path = "crates/wait_queue_macros" } paste = "=1.0.14" slabmalloc = { path = "crates/rust-slabmalloc" } log = "0.4.21" +kprobe = { path = "crates/kprobe" } +xarray = "0.1.0" lru = "0.12.3" +rbpf = { path = "crates/rbpf" } +printf-compat = { version = "0.1.1", default-features = false } # target为x86_64时，使用下面的依赖 [target.'cfg(target_arch = "x86_64")'.dependencies] mini-backtrace = { git = "https://git.mirrors.dragonos.org.cn/DragonOS-Community/mini-backtrace.git", rev = "e0b1d90940" } diff --git a/kernel/Makefile b/kernel/Makefile index be85d3e7b..90543aafe 100644 --- a/kernel/Makefile +++ b/kernel/Makefile @@ -43,5 +43,7 @@ endif test: # 测试内核库 - RUSTFLAGS="$(RUSTFLAGS)" cargo +nightly-2024-07-23 test --workspace --exclude dragonos_kernel + RUSTFLAGS="$(RUSTFLAGS)" cargo +nightly-2024-07-23 test --workspace --exclude dragonos_kernel rbpf +test-rbpf: + cd crates/rbpf && RUSTFLAGS="$(RUSTFLAGS)" cargo +nightly-2024-07-23 test --features=std,user,cranelift \ No newline at end of file diff --git a/kernel/crates/kprobe/Cargo.toml b/kernel/crates/kprobe/Cargo.toml new file mode 100644 index 000000000..063591aa1 --- /dev/null +++ b/kernel/crates/kprobe/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "kprobe" +version = "0.1.0" +edition = "2021" + +[dependencies] +log = "0.4.21" + +[target.'cfg(target_arch = "x86_64")'.dependencies] +yaxpeax-x86 = { version = "2", default-features = false, features = ["fmt"] } +yaxpeax-arch = { version = "0", default-features = false } diff --git a/kernel/crates/kprobe/src/arch/loongarch64/mod.rs b/kernel/crates/kprobe/src/arch/loongarch64/mod.rs new file mode 100644 index 000000000..263e7cd7c --- /dev/null +++ b/kernel/crates/kprobe/src/arch/loongarch64/mod.rs @@ -0,0 +1,112 @@ +use alloc::sync::Arc; +use core::ops::{Deref, DerefMut}; + +use crate::{KprobeBasic, KprobeBuilder, KprobeOps}; + +const BRK_KPROBE_BP: u64 = 10; +const BRK_KPROBE_SSTEPBP: u64 = 11; +const EBREAK_INST: u32 = 0x002a0000; + +#[derive(Debug)] +pub struct Kprobe { + basic: KprobeBasic, + point: Arc, +} +#[derive(Debug)] +pub struct LA64KprobePoint { + addr: usize, + inst_tmp: [u8; 8], +} + +impl Deref for Kprobe { + type Target = KprobeBasic; + + fn deref(&self) -> &Self::Target { + &self.basic + } +} + +impl DerefMut for Kprobe { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.basic + } +} + +impl Kprobe { + pub fn probe_point(&self) -> &Arc { + &self.point + } +} + +impl Drop for LA64KprobePoint { + fn drop(&mut self) { + let address = self.addr; + let inst_tmp_ptr = self.inst_tmp.as_ptr() as usize; + let inst_32 = unsafe { core::ptr::read(inst_tmp_ptr as *const u32) }; + unsafe { + core::ptr::write(address as *mut u32, inst_32); + } + log::trace!( + "Kprobe::uninstall: address: {:#x}, old_instruction: {:?}", + address, + inst_32 + ); + } +} + +impl KprobeBuilder { + pub fn install(self) -> (Kprobe, Arc) { + let probe_point = match &self.probe_point { + Some(point) => point.clone(), + None => self.replace_inst(), + }; + let kprobe = Kprobe { + basic: KprobeBasic::from(self), + point: probe_point.clone(), + }; + (kprobe, probe_point) + } + /// # 安装kprobe + /// + /// 不同的架构下需要保存原指令，然后替换为断点指令 + fn replace_inst(&self) -> Arc { + let address = self.symbol_addr + self.offset; + let point = LA64KprobePoint { + addr: address, + inst_tmp: [0u8; 8], + }; + let inst_tmp_ptr = point.inst_tmp.as_ptr() as usize; + let inst_32 = unsafe { core::ptr::read(address as *const u32) }; + unsafe { + core::ptr::write(address as *mut u32, EBREAK_INST); + // inst_32 :0-32 + // ebreak :32-64 + core::ptr::write(inst_tmp_ptr as *mut u32, inst_32); + core::ptr::write((inst_tmp_ptr + 4) as *mut u32, EBREAK_INST); + } + log::trace!( + "Kprobe::install: address: {:#x}, func_name: {:?}, opcode: {:x?}", + address, + self.symbol, + inst_32 + ); + } +} + +impl KprobeOps for LA64KprobePoint { + fn return_address(&self) -> usize { + self.addr + 4 + } + + fn single_step_address(&self) -> usize { + self.inst_tmp.as_ptr() as usize + } + + fn debug_address(&self) -> usize { + self.inst_tmp.as_ptr() as usize + 4 + } + + fn break_address(&self) -> usize { + self.addr + } +} diff --git a/kernel/crates/kprobe/src/arch/mod.rs b/kernel/crates/kprobe/src/arch/mod.rs new file mode 100644 index 000000000..27abd5593 --- /dev/null +++ b/kernel/crates/kprobe/src/arch/mod.rs @@ -0,0 +1,211 @@ +use alloc::boxed::Box; +use alloc::string::String; +use alloc::sync::Arc; +use core::{any::Any, fmt::Debug}; + +#[cfg(target_arch = "loongarch64")] +mod loongarch64; +#[cfg(target_arch = "riscv64")] +mod rv64; +#[cfg(target_arch = "x86_64")] +mod x86; + +#[cfg(target_arch = "loongarch64")] +pub use loongarch64::*; +#[cfg(target_arch = "riscv64")] +pub use rv64::*; +#[cfg(target_arch = "x86_64")] +pub use x86::*; + +#[cfg(target_arch = "x86_64")] +pub type KprobePoint = X86KprobePoint; +#[cfg(target_arch = "riscv64")] +pub type KprobePoint = Rv64KprobePoint; +#[cfg(target_arch = "loongarch64")] +pub type KprobePoint = LA64KprobePoint; + +pub trait ProbeArgs: Send { + /// 用于使用者转换到特定架构下的TrapFrame + fn as_any(&self) -> &dyn Any; + /// 返回导致break异常的地址 + fn break_address(&self) -> usize; + /// 返回导致单步执行异常的地址 + fn debug_address(&self) -> usize; +} + +pub trait KprobeOps: Send { + /// # 返回探测点的下一条指令地址 + /// + /// 执行流需要回到正常的路径中，在执行完探测点的指令后，需要返回到下一条指令 + fn return_address(&self) -> usize; + /// # 返回单步执行的指令地址 + /// + /// 通常探测点的处的原指令被保存在一个数组当中。根据架构的不同, 在保存的指令后面，可能会填充必要的指令。 + /// 例如x86架构下支持单步执行的特性， 而其它架构下通常没有，因此我们使用break异常来进行模拟，所以会填充 + /// 一条断点指令。 + fn single_step_address(&self) -> usize; + /// # 返回单步执行指令触发异常的地址 + /// + /// 其值等于`single_step_address`的值加上探测点指令的长度 + fn debug_address(&self) -> usize; + /// # 返回设置break断点的地址 + /// + /// 其值与探测点地址相等 + fn break_address(&self) -> usize; +} + +struct ProbeHandler { + func: fn(&dyn ProbeArgs), +} + +impl ProbeHandler { + pub fn new(func: fn(&dyn ProbeArgs)) -> Self { + ProbeHandler { func } + } + /// 调用探测点处理函数 + pub fn call(&self, trap_frame: &dyn ProbeArgs) { + (self.func)(trap_frame); + } +} + +pub struct KprobeBuilder { + symbol: Option, + symbol_addr: usize, + offset: usize, + pre_handler: ProbeHandler, + post_handler: ProbeHandler, + fault_handler: Option, + event_callback: Option>, + probe_point: Option>, + enable: bool, +} + +pub trait EventCallback: Send { + fn call(&self, trap_frame: &dyn ProbeArgs); +} + +impl KprobeBuilder { + pub fn new( + symbol: Option, + symbol_addr: usize, + offset: usize, + pre_handler: fn(&dyn ProbeArgs), + post_handler: fn(&dyn ProbeArgs), + enable: bool, + ) -> Self { + KprobeBuilder { + symbol, + symbol_addr, + offset, + pre_handler: ProbeHandler::new(pre_handler), + post_handler: ProbeHandler::new(post_handler), + event_callback: None, + fault_handler: None, + probe_point: None, + enable, + } + } + + pub fn with_fault_handler(mut self, func: fn(&dyn ProbeArgs)) -> Self { + self.fault_handler = Some(ProbeHandler::new(func)); + self + } + + pub fn with_probe_point(mut self, point: Arc) -> Self { + self.probe_point = Some(point); + self + } + + pub fn with_event_callback(mut self, event_callback: Box) -> Self { + self.event_callback = Some(event_callback); + self + } + + /// 获取探测点的地址 + /// + /// 探测点的地址 == break指令的地址 + pub fn probe_addr(&self) -> usize { + self.symbol_addr + self.offset + } +} + +pub struct KprobeBasic { + symbol: Option, + symbol_addr: usize, + offset: usize, + pre_handler: ProbeHandler, + post_handler: ProbeHandler, + fault_handler: ProbeHandler, + event_callback: Option>, + enable: bool, +} + +pub trait CallBackFunc: Send + Sync { + fn call(&self, trap_frame: &dyn ProbeArgs); +} + +impl Debug for KprobeBasic { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.debug_struct("Kprobe") + .field("symbol", &self.symbol) + .field("symbol_addr", &self.symbol_addr) + .field("offset", &self.offset) + .finish() + } +} + +impl KprobeBasic { + pub fn call_pre_handler(&self, trap_frame: &dyn ProbeArgs) { + self.pre_handler.call(trap_frame); + } + + pub fn call_post_handler(&self, trap_frame: &dyn ProbeArgs) { + self.post_handler.call(trap_frame); + } + + pub fn call_fault_handler(&self, trap_frame: &dyn ProbeArgs) { + self.fault_handler.call(trap_frame); + } + + pub fn call_event_callback(&self, trap_frame: &dyn ProbeArgs) { + if let Some(ref call_back) = self.event_callback { + call_back.call(trap_frame); + } + } + + pub fn update_event_callback(&mut self, callback: Box) { + self.event_callback = Some(callback); + } + + pub fn disable(&mut self) { + self.enable = false; + } + + pub fn enable(&mut self) { + self.enable = true; + } + + pub fn is_enabled(&self) -> bool { + self.enable + } + /// 返回探测点的函数名称 + pub fn symbol(&self) -> Option<&str> { + self.symbol.as_deref() + } +} + +impl From for KprobeBasic { + fn from(value: KprobeBuilder) -> Self { + let fault_handler = value.fault_handler.unwrap_or(ProbeHandler::new(|_| {})); + KprobeBasic { + symbol: value.symbol, + symbol_addr: value.symbol_addr, + offset: value.offset, + pre_handler: value.pre_handler, + post_handler: value.post_handler, + event_callback: value.event_callback, + fault_handler, + enable: value.enable, + } + } +} diff --git a/kernel/crates/kprobe/src/arch/rv64/mod.rs b/kernel/crates/kprobe/src/arch/rv64/mod.rs new file mode 100644 index 000000000..b73deac6d --- /dev/null +++ b/kernel/crates/kprobe/src/arch/rv64/mod.rs @@ -0,0 +1,157 @@ +use alloc::sync::Arc; +use core::{ + arch::riscv64::sfence_vma_all, + fmt::Debug, + ops::{Deref, DerefMut}, +}; + +use crate::{KprobeBasic, KprobeBuilder, KprobeOps}; +const EBREAK_INST: u32 = 0x00100073; // ebreak +const C_EBREAK_INST: u32 = 0x9002; // c.ebreak +const INSN_LENGTH_MASK: u16 = 0x3; +const INSN_LENGTH_32: u16 = 0x3; + +#[derive(Debug)] +pub struct Kprobe { + basic: KprobeBasic, + point: Arc, +} + +#[derive(Debug)] +enum OpcodeTy { + Inst16(u16), + Inst32(u32), +} +#[derive(Debug)] +pub struct Rv64KprobePoint { + addr: usize, + old_instruction: OpcodeTy, + inst_tmp: [u8; 8], +} + +impl Deref for Kprobe { + type Target = KprobeBasic; + + fn deref(&self) -> &Self::Target { + &self.basic + } +} + +impl DerefMut for Kprobe { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.basic + } +} + +impl Kprobe { + pub fn probe_point(&self) -> &Arc { + &self.point + } +} + +impl Drop for Rv64KprobePoint { + fn drop(&mut self) { + let address = self.addr; + match self.old_instruction { + OpcodeTy::Inst16(inst_16) => unsafe { + core::ptr::write(address as *mut u16, inst_16); + }, + OpcodeTy::Inst32(inst_32) => unsafe { + core::ptr::write(address as *mut u32, inst_32); + }, + } + unsafe { + sfence_vma_all(); + } + log::trace!( + "Kprobe::uninstall: address: {:#x}, old_instruction: {:?}", + address, + self.old_instruction + ); + } +} + +impl KprobeBuilder { + pub fn install(self) -> (Kprobe, Arc) { + let probe_point = match &self.probe_point { + Some(point) => point.clone(), + None => self.replace_inst(), + }; + let kprobe = Kprobe { + basic: KprobeBasic::from(self), + point: probe_point.clone(), + }; + (kprobe, probe_point) + } + /// # 安装kprobe + /// + /// 不同的架构下需要保存原指令，然后替换为断点指令 + fn replace_inst(&self) -> Arc { + let address = self.symbol_addr + self.offset; + let inst_16 = unsafe { core::ptr::read(address as *const u16) }; + // See https://elixir.bootlin.com/linux/v6.10.2/source/arch/riscv/kernel/probes/kprobes.c#L68 + let is_inst_16 = if (inst_16 & INSN_LENGTH_MASK) == INSN_LENGTH_32 { + false + } else { + true + }; + let mut point = Rv64KprobePoint { + old_instruction: OpcodeTy::Inst16(0), + inst_tmp: [0; 8], + addr: address, + }; + let inst_tmp_ptr = point.inst_tmp.as_ptr() as usize; + if is_inst_16 { + point.old_instruction = OpcodeTy::Inst16(inst_16); + unsafe { + core::ptr::write(address as *mut u16, C_EBREAK_INST as u16); + // inst_16 :0-16 + // c.ebreak:16-32 + core::ptr::write(inst_tmp_ptr as *mut u16, inst_16); + core::ptr::write((inst_tmp_ptr + 2) as *mut u16, C_EBREAK_INST as u16); + } + } else { + let inst_32 = unsafe { core::ptr::read(address as *const u32) }; + point.old_instruction = OpcodeTy::Inst32(inst_32); + unsafe { + core::ptr::write(address as *mut u32, EBREAK_INST); + // inst_32 :0-32 + // ebreak :32-64 + core::ptr::write(inst_tmp_ptr as *mut u32, inst_32); + core::ptr::write((inst_tmp_ptr + 4) as *mut u32, EBREAK_INST); + } + } + unsafe { + sfence_vma_all(); + } + log::trace!( + "Kprobe::install: address: {:#x}, func_name: {:?}, opcode: {:x?}", + address, + self.symbol, + point.old_instruction + ); + Arc::new(point) + } +} + +impl KprobeOps for Rv64KprobePoint { + fn return_address(&self) -> usize { + let address = self.addr; + match self.old_instruction { + OpcodeTy::Inst16(_) => address + 2, + OpcodeTy::Inst32(_) => address + 4, + } + } + fn single_step_address(&self) -> usize { + self.inst_tmp.as_ptr() as usize + } + fn debug_address(&self) -> usize { + match self.old_instruction { + OpcodeTy::Inst16(_) => self.inst_tmp.as_ptr() as usize + 2, + OpcodeTy::Inst32(_) => self.inst_tmp.as_ptr() as usize + 4, + } + } + fn break_address(&self) -> usize { + self.addr + } +} diff --git a/kernel/crates/kprobe/src/arch/x86/mod.rs b/kernel/crates/kprobe/src/arch/x86/mod.rs new file mode 100644 index 000000000..e8fb83f69 --- /dev/null +++ b/kernel/crates/kprobe/src/arch/x86/mod.rs @@ -0,0 +1,135 @@ +use crate::{KprobeBasic, KprobeBuilder, KprobeOps}; +use alloc::string::ToString; +use alloc::sync::Arc; +use core::{ + fmt::Debug, + ops::{Deref, DerefMut}, +}; +use yaxpeax_arch::LengthedInstruction; + +const EBREAK_INST: u8 = 0xcc; // x86_64: 0xcc +const MAX_INSTRUCTION_SIZE: usize = 15; // x86_64 max instruction length + +pub struct Kprobe { + basic: KprobeBasic, + point: Arc, +} + +#[derive(Debug)] +pub struct X86KprobePoint { + addr: usize, + old_instruction: [u8; MAX_INSTRUCTION_SIZE], + old_instruction_len: usize, +} + +impl Drop for X86KprobePoint { + fn drop(&mut self) { + let address = self.addr; + unsafe { + core::ptr::copy( + self.old_instruction.as_ptr(), + address as *mut u8, + self.old_instruction_len, + ); + core::arch::x86_64::_mm_mfence(); + } + let decoder = yaxpeax_x86::amd64::InstDecoder::default(); + let inst = decoder.decode_slice(&self.old_instruction).unwrap(); + log::trace!( + "Kprobe::uninstall: address: {:#x}, old_instruction: {:?}", + address, + inst.to_string() + ); + } +} + +impl Debug for Kprobe { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.debug_struct("Kprobe") + .field("basic", &self.basic) + .field("point", &self.point) + .finish() + } +} + +impl Deref for Kprobe { + type Target = KprobeBasic; + + fn deref(&self) -> &Self::Target { + &self.basic + } +} + +impl DerefMut for Kprobe { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.basic + } +} + +impl KprobeBuilder { + pub fn install(self) -> (Kprobe, Arc) { + let probe_point = match &self.probe_point { + Some(point) => point.clone(), + None => self.replace_inst(), + }; + let kprobe = Kprobe { + basic: KprobeBasic::from(self), + point: probe_point.clone(), + }; + (kprobe, probe_point) + } + /// # 安装kprobe + /// + /// 不同的架构下需要保存原指令，然后替换为断点指令 + fn replace_inst(&self) -> Arc { + let address = self.symbol_addr + self.offset; + let mut inst_tmp = [0u8; MAX_INSTRUCTION_SIZE]; + unsafe { + core::ptr::copy( + address as *const u8, + inst_tmp.as_mut_ptr(), + MAX_INSTRUCTION_SIZE, + ); + } + let decoder = yaxpeax_x86::amd64::InstDecoder::default(); + let inst = decoder.decode_slice(&inst_tmp).unwrap(); + let len = inst.len().to_const(); + log::trace!("inst: {:?}, len: {:?}", inst.to_string(), len); + let point = Arc::new(X86KprobePoint { + addr: address, + old_instruction: inst_tmp, + old_instruction_len: len as usize, + }); + unsafe { + core::ptr::write_volatile(address as *mut u8, EBREAK_INST); + core::arch::x86_64::_mm_mfence(); + } + log::trace!( + "Kprobe::install: address: {:#x}, func_name: {:?}", + address, + self.symbol + ); + point + } +} + +impl Kprobe { + pub fn probe_point(&self) -> &Arc { + &self.point + } +} + +impl KprobeOps for X86KprobePoint { + fn return_address(&self) -> usize { + self.addr + self.old_instruction_len + } + fn single_step_address(&self) -> usize { + self.old_instruction.as_ptr() as usize + } + fn debug_address(&self) -> usize { + self.old_instruction.as_ptr() as usize + self.old_instruction_len + } + fn break_address(&self) -> usize { + self.addr + } +} diff --git a/kernel/crates/kprobe/src/lib.rs b/kernel/crates/kprobe/src/lib.rs new file mode 100644 index 000000000..7e871c6de --- /dev/null +++ b/kernel/crates/kprobe/src/lib.rs @@ -0,0 +1,7 @@ +#![cfg_attr(target_arch = "riscv64", feature(riscv_ext_intrinsics))] +#![no_std] +extern crate alloc; + +mod arch; + +pub use arch::*; diff --git a/kernel/crates/rbpf/.appveyor.yml b/kernel/crates/rbpf/.appveyor.yml new file mode 100644 index 000000000..c8ea114a4 --- /dev/null +++ b/kernel/crates/rbpf/.appveyor.yml @@ -0,0 +1,21 @@ +version: 1.0.{build} +branches: + only: + - main +os: + - Visual Studio 2015 +clone_depth: 1 +configuration: + - Debug +platform: + - x64 +environment: + matrix: + - TOOLCHAIN_VERSION: 14.0 + RUST: 1.76.0 + - TOOLCHAIN_VERSION: 14.0 + RUST: beta + - TOOLCHAIN_VERSION: 14.0 + RUST: nightly + +build_script: mk/appveyor.bat diff --git a/kernel/crates/rbpf/.gitignore b/kernel/crates/rbpf/.gitignore new file mode 100644 index 000000000..a9d37c560 --- /dev/null +++ b/kernel/crates/rbpf/.gitignore @@ -0,0 +1,2 @@ +target +Cargo.lock diff --git a/kernel/crates/rbpf/Cargo.toml b/kernel/crates/rbpf/Cargo.toml new file mode 100644 index 000000000..1a711ac97 --- /dev/null +++ b/kernel/crates/rbpf/Cargo.toml @@ -0,0 +1,78 @@ +[package] + +# Project metadata +name = "rbpf" +version = "0.2.0" +authors = ["Quentin "] + +# Additional metadata for packaging +description = "Virtual machine and JIT compiler for eBPF programs" +repository = "https://github.com/qmonnet/rbpf" +readme = "README.md" +keywords = ["BPF", "eBPF", "interpreter", "JIT", "filtering"] +license = "Apache-2.0/MIT" +edition = "2021" + +# Packaging directives +include = [ + "src/**", + "examples/**", + "tests/**", + "bench/**", + "LICENSE*", + "Cargo.toml", +] + +[dependencies] + +# Default features (std) are disabled so that the dependencies don't pull in the +# standard library when the crate is compiled for no_std +byteorder = { version = "1.2", default-features = false } +log = {version = "0.4.21", default-features = false } +combine = { version = "4.6", default-features = false } + +# Optional Dependencies when using the standard library +libc = { version = "0.2", optional = true } +time = { version = "0.2", optional = true } + +# Optional Dependencies for the CraneLift JIT +cranelift-codegen = { version = "0.99", optional = true } +cranelift-frontend = { version = "0.99", optional = true } +cranelift-jit = { version = "0.99", optional = true } +cranelift-native = { version = "0.99", optional = true } +cranelift-module = { version = "0.99", optional = true } + +[dev-dependencies] + +elf = "0.0.10" +json = "0.11" +hex = "0.4.3" + +[features] +#default = ["std", "user", "cranelift"] +cargo-clippy = [] +std = ["dep:time", "dep:libc", "combine/std"] +cranelift = [ + "dep:cranelift-codegen", + "dep:cranelift-frontend", + "dep:cranelift-jit", + "dep:cranelift-native", + "dep:cranelift-module", +] +user = [] + +# Examples that depend on the standard library should be disabled when +# testing the `no_std` configuration. +[[example]] +name = "disassemble" +required-features = ["std"] + +[[example]] +name = "uptime" +required-features = ["std"] + +[[example]] +name = "to_json" + +[[example]] +name = "rbpf_plugin" diff --git a/kernel/crates/rbpf/LICENSE-APACHE b/kernel/crates/rbpf/LICENSE-APACHE new file mode 100644 index 000000000..d64569567 --- /dev/null +++ b/kernel/crates/rbpf/LICENSE-APACHE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/kernel/crates/rbpf/LICENSE-MIT b/kernel/crates/rbpf/LICENSE-MIT new file mode 100644 index 000000000..661a705a2 --- /dev/null +++ b/kernel/crates/rbpf/LICENSE-MIT @@ -0,0 +1,25 @@ +Copyright (c) 2016 6WIND S.A. + +Permission is hereby granted, free of charge, to any +person obtaining a copy of this software and associated +documentation files (the "Software"), to deal in the +Software without restriction, including without +limitation the rights to use, copy, modify, merge, +publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software +is furnished to do so, subject to the following +conditions: + +The above copyright notice and this permission notice +shall be included in all copies or substantial portions +of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF +ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED +TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. diff --git a/kernel/crates/rbpf/README.md b/kernel/crates/rbpf/README.md new file mode 100644 index 000000000..e2dc8ce7b --- /dev/null +++ b/kernel/crates/rbpf/README.md @@ -0,0 +1,743 @@ +# rbpf + + + + + + +Rust (user-space) virtual machine for eBPF + +[![Build Status](https://github.com/qmonnet/rbpf/actions/workflows/test.yaml/badge.svg)](https://github.com/qmonnet/rbpf/actions/workflows/test.yaml) +[![Build status](https://ci.appveyor.com/api/projects/status/ia74coeuhxtrcvsk/branch/main?svg=true)](https://ci.appveyor.com/project/qmonnet/rbpf/branch/main) +[![Coverage Status](https://coveralls.io/repos/github/qmonnet/rbpf/badge.svg?branch=main)](https://coveralls.io/github/qmonnet/rbpf?branch=main) +[![Crates.io](https://img.shields.io/crates/v/rbpf.svg)](https://crates.io/crates/rbpf) + +* [Description](#description) +* [Link to the crate](#link-to-the-crate) +* [API](#api) +* [Example uses](#example-uses) +* [Building eBPF programs](#building-ebpf-programs) +* [Build Features](#build-features) +* [Feedback welcome!](#feedback-welcome) +* [Questions / Answers](#questions--answers) +* [Caveats](#caveats) +* [_To do_ list](#to-do-list) +* [License](#license) +* [Inspired by](#inspired-by) +* [Other resources](#other-resources) + +## Description + +This crate contains a virtual machine for eBPF program execution. BPF, as in +_Berkeley Packet Filter_, is an assembly-like language initially developed for +BSD systems, in order to filter packets in the kernel with tools such as +tcpdump so as to avoid useless copies to user-space. It was ported to Linux, +where it evolved into eBPF (_extended_ BPF), a faster version with more +features. While BPF programs are originally intended to run in the kernel, the +virtual machine of this crate enables running it in user-space applications; +it contains an interpreter, an x86_64 JIT-compiler for eBPF programs, as well as +a disassembler. + +It is based on Rich Lane's [uBPF software](https://github.com/iovisor/ubpf/), +which does nearly the same, but is written in C. + +The crate is supposed to compile and run on Linux, MacOS X, and Windows, +although the JIT-compiler does not work with Windows at this time. + +## Link to the crate + +This crate is available from [crates.io](https://crates.io/crates/rbpf), so it +should work out of the box by adding it as a dependency in your `Cargo.toml` +file: + +```toml +[dependencies] +rbpf = "0.2.0" +``` + +You can also use the development version from this GitHub repository. This +should be as simple as putting this inside your `Cargo.toml`: + +```toml +[dependencies] +rbpf = { git = "https://github.com/qmonnet/rbpf" } +``` + +Of course, if you prefer, you can clone it locally, possibly hack the crate, +and then indicate the path of your local version in `Cargo.toml`: + +```toml +[dependencies] +rbpf = { path = "path/to/rbpf" } +``` + +Then indicate in your source code that you want to use the crate: + +```rust,ignore +extern crate rbpf; +``` + +## API + +The API is pretty well documented inside the source code. You should also be +able to access [an online version of the documentation from +here](https://docs.rs/rbpf/), automatically generated from the +[crates.io](https://crates.io/crates/rbpf) version (may not be up-to-date with +the main branch). [Examples](../../tree/main/examples) and [unit +tests](../../tree/main/tests) should also prove helpful. Here is a summary of +how to use the crate. + +Here are the steps to follow to run an eBPF program with rbpf: + +1. Create a virtual machine. There are several kinds of machines, we will come + back on this later. When creating the VM, pass the eBPF program as an + argument to the constructor. +2. If you want to use some helper functions, register them into the virtual + machine. +3. If you want a JIT-compiled program, compile it. +4. Execute your program: either run the interpreter or call the JIT-compiled + function. + +eBPF has been initially designed to filter packets (now it has some other hooks +in the Linux kernel, such as kprobes, but this is not covered by rbpf). As a +consequence, most of the load and store instructions of the program are +performed on a memory area representing the packet data. However, in the Linux +kernel, the eBPF program does not immediately access this data area: initially, +it has access to a C `struct sk_buff` instead, which is a buffer containing +metadata about the packet—including memory addresses of the beginning and of +the end of the packet data area. So the program first loads those pointers from +the `sk_buff`, and then can access the packet data. + +This behavior can be replicated with rbpf, but it is not mandatory. For this +reason, we have several structs representing different kinds of virtual +machines: + +* `struct EbpfVmMbuffer` mimics the kernel. When the program is run, the + address provided to its first eBPF register will be the address of a metadata + buffer provided by the user, and that is expected to contain pointers to the + start and the end of the packet data memory area. + +* `struct EbpfVmFixedMbuff` has one purpose: enabling the execution of programs + created to be compatible with the kernel, while saving the effort to manually + handle the metadata buffer for the user. In fact, this struct has a static + internal buffer that is passed to the program. The user has to indicate the + offset values at which the eBPF program expects to find the start and the end + of packet data in the buffer. On calling the function that runs the program + (JITted or not), the struct automatically updates the addresses in this + static buffer, at the appointed offsets, for the start and the end of the + packet data the program is called upon. + +* `struct EbpfVmRaw` is for programs that want to run directly on packet data. + No metadata buffer is involved, the eBPF program directly receives the + address of the packet data in its first register. This is the behavior of + uBPF. + +* `struct EbpfVmNoData` does not take any data. The eBPF program takes no + argument whatsoever and its return value is deterministic. Not so sure there + is a valid use case for that, but if nothing else, this is very useful for + unit tests. + +All these structs implement the same public functions: + +```rust,ignore +// called with EbpfVmMbuff:: prefix +pub fn new(prog: &'a [u8]) -> Result, Error> + +// called with EbpfVmFixedMbuff:: prefix +pub fn new(prog: &'a [u8], + data_offset: usize, + data_end_offset: usize) -> Result, Error> + +// called with EbpfVmRaw:: prefix +pub fn new(prog: &'a [u8]) -> Result, Error> + +// called with EbpfVmNoData:: prefix +pub fn new(prog: &'a [u8]) -> Result, Error> +``` + +This is used to create a new instance of a VM. The return type is dependent of +the struct from which the function is called. For instance, +`rbpf::EbpfVmRaw::new(Some(my_program))` would return an instance of `struct +rbpf::EbpfVmRaw` (wrapped in a `Result`). When a program is loaded, it is +checked with a very simple verifier (nothing close to the one for Linux +kernel). Users are also able to replace it with a custom verifier. + +For `struct EbpfVmFixedMbuff`, two additional arguments must be passed to the +constructor: `data_offset` and `data_end_offset`. They are the offset (byte +number) at which the pointers to the beginning and to the end, respectively, of +the memory area of packet data are to be stored in the internal metadata buffer +each time the program is executed. Other structs do not use this mechanism and +do not need those offsets. + +```rust,ignore +// for struct EbpfVmMbuff, struct EbpfVmRaw and struct EbpfVmRawData +pub fn set_program(&mut self, prog: &'a [u8]) -> Result<(), Error> + +// for struct EbpfVmFixedMbuff +pub fn set_program(&mut self, prog: &'a [u8], + data_offset: usize, + data_end_offset: usize) -> Result<(), Error> +``` + +You can use for example `my_vm.set_program(my_program);` to change the loaded +program after the VM instance creation. This program is checked with the +verifier attached to the VM. The verifying function of the VM can be changed at +any moment. + +```rust,ignore +pub type Verifier = fn(prog: &[u8]) -> Result<(), Error>; + +pub fn set_verifier(&mut self, + verifier: Verifier) -> Result<(), Error> +``` + +Note that if a program has already been loaded into the VM, setting a new +verifier also immediately runs it on the loaded program. However, the verifier +is not run if no program has been loaded (if `None` was passed to the `new()` +method when creating the VM). + +```rust,ignore +pub type Helper = fn (u64, u64, u64, u64, u64) -> u64; + +pub fn register_helper(&mut self, + key: u32, + function: Helper) -> Result<(), Error> +``` + +This function is used to register a helper function. The VM stores its +registers in a hashmap, so the key can be any `u32` value you want. It may be +useful for programs that should be compatible with the Linux kernel and +therefore must use specific helper numbers. + +```rust,ignore +// for struct EbpfVmMbuff +pub fn execute_program(&self, + mem: &'a mut [u8], + mbuff: &'a mut [u8]) -> Result<(u64), Error> + +// for struct EbpfVmFixedMbuff and struct EbpfVmRaw +pub fn execute_program(&self, + mem: &'a mut [u8]) -> Result<(u64), Error> + +// for struct EbpfVmNoData +pub fn execute_program(&self) -> Result<(u64), Error> +``` + +Interprets the loaded program. The function takes a reference to the packet +data and the metadata buffer, or only to the packet data, or nothing at all, +depending on the kind of the VM used. The value returned is the result of the +eBPF program. + +```rust,ignore +pub fn jit_compile(&mut self) -> Result<(), Error> +``` + +JIT-compile the loaded program, for x86_64 architecture. If the program is to +use helper functions, they must be registered into the VM before this function +is called. The generated assembly function is internally stored in the VM. + +```rust,ignore +// for struct EbpfVmMbuff +pub unsafe fn execute_program_jit(&self, mem: &'a mut [u8], + mbuff: &'a mut [u8]) -> Result<(u64), Error> + +// for struct EbpfVmFixedMbuff and struct EbpfVmRaw +pub unsafe fn execute_program_jit(&self, mem: &'a mut [u8]) -> Result<(u64), Error> + +// for struct EbpfVmNoData +pub unsafe fn execute_program_jit(&self) -> Result<(u64), Error> +``` + +Calls the JIT-compiled program. The arguments to provide are the same as for +`execute_program()`, again depending on the kind of VM that is used. The result of +the JIT-compiled program should be the same as with the interpreter, but it +should run faster. Note that if errors occur during the program execution, the +JIT-compiled version does not handle it as well as the interpreter, and the +program may crash. For this reason, the functions are marked as `unsafe`. + +## Example uses + +### Simple example + +This comes from the unit test `test_vm_add`. + +```rust +extern crate rbpf; + +fn main() { + + // This is the eBPF program, in the form of bytecode instructions. + let prog = &[ + 0xb4, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // mov32 r0, 0 + 0xb4, 0x01, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, // mov32 r1, 2 + 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, // add32 r0, 1 + 0x0c, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // add32 r0, r1 + 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 // exit + ]; + + // Instantiate a struct EbpfVmNoData. This is an eBPF VM for programs that + // takes no packet data in argument. + // The eBPF program is passed to the constructor. + let vm = rbpf::EbpfVmNoData::new(Some(prog)).unwrap(); + + // Execute (interpret) the program. No argument required for this VM. + assert_eq!(vm.execute_program().unwrap(), 0x3); +} +``` + +### With JIT, on packet data + +This comes from the unit test `test_jit_ldxh`. + +```rust +extern crate rbpf; + +fn main() { + let prog = &[ + 0x71, 0x10, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, // ldxh r0, [r1+2] + 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 // exit + ]; + + // Let's use some data. + let mem = &mut [ + 0xaa, 0xbb, 0x11, 0xcc, 0xdd + ]; + + // This is an eBPF VM for programs reading from a given memory area (it + // directly reads from packet data) + let mut vm = rbpf::EbpfVmRaw::new(Some(prog)).unwrap(); + + #[cfg(any(windows, not(feature = "std")))] { + assert_eq!(vm.execute_program(mem).unwrap(), 0x11); + } + #[cfg(all(not(windows), feature = "std"))] { + // This time we JIT-compile the program. + vm.jit_compile().unwrap(); + + // Then we execute it. For this kind of VM, a reference to the packet + // data must be passed to the function that executes the program. + unsafe { assert_eq!(vm.execute_program_jit(mem).unwrap(), 0x11); } + } +} +``` +### Using a metadata buffer + +This comes from the unit test `test_jit_mbuff` and derives from the unit test +`test_jit_ldxh`. + +```rust +extern crate rbpf; + +fn main() { + let prog = &[ + // Load mem from mbuff at offset 8 into R1 + 0x79, 0x11, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, + // ldhx r1[2], r0 + 0x69, 0x10, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 + ]; + let mem = &mut [ + 0xaa, 0xbb, 0x11, 0x22, 0xcc, 0xdd + ]; + + // Just for the example we create our metadata buffer from scratch, and + // we store the pointers to packet data start and end in it. + let mut mbuff = &mut [0u8; 32]; + unsafe { + let mut data = mbuff.as_ptr().offset(8) as *mut u64; + let mut data_end = mbuff.as_ptr().offset(24) as *mut u64; + *data = mem.as_ptr() as u64; + *data_end = mem.as_ptr() as u64 + mem.len() as u64; + } + + // This eBPF VM is for program that use a metadata buffer. + let mut vm = rbpf::EbpfVmMbuff::new(Some(prog)).unwrap(); + + #[cfg(any(windows, not(feature = "std")))] { + assert_eq!(vm.execute_program(mem, mbuff).unwrap(), 0x2211); + } + #[cfg(all(not(windows), feature = "std"))] { + // Here again we JIT-compile the program. + vm.jit_compile().unwrap(); + + // Here we must provide both a reference to the packet data, and to the + // metadata buffer we use. + unsafe { + assert_eq!(vm.execute_program_jit(mem, mbuff).unwrap(), 0x2211); + } + } +} +``` + +### Loading code from an object file; and using a virtual metadata buffer + +This comes from unit test `test_vm_block_port`. + +This example requires the following additional crates, you may have to add them +to your `Cargo.toml` file. + +```toml +[dependencies] +rbpf = "0.2.0" +elf = "0.0.10" +``` + +It also uses a kind of VM that uses an internal buffer used to simulate the +`sk_buff` used by eBPF programs in the kernel, without having to manually +create a new buffer for each packet. It may be useful for programs compiled for +the kernel and that assumes the data they receive is a `sk_buff` pointing to +the packet data start and end addresses. So here we just provide the offsets at +which the eBPF program expects to find those pointers, and the VM handles the +buffer update so that we only have to provide a reference to the packet data +for each run of the program. + +```rust +extern crate elf; +use std::path::PathBuf; + +extern crate rbpf; +use rbpf::helpers; + +fn main() { + // Load a program from an ELF file, e.g. compiled from C to eBPF with + // clang/LLVM. Some minor modification to the bytecode may be required. + let filename = "examples/load_elf__block_a_port.elf"; + + let path = PathBuf::from(filename); + let file = match elf::File::open_path(&path) { + Ok(f) => f, + Err(e) => panic!("Error: {:?}", e), + }; + + // Here we assume the eBPF program is in the ELF section called + // ".classifier". + let text_scn = match file.get_section(".classifier") { + Some(s) => s, + None => panic!("Failed to look up .classifier section"), + }; + + let prog = &text_scn.data; + + // This is our data: a real packet, starting with Ethernet header + let packet = &mut [ + 0x01, 0x23, 0x45, 0x67, 0x89, 0xab, + 0xfe, 0xdc, 0xba, 0x98, 0x76, 0x54, + 0x08, 0x00, // ethertype + 0x45, 0x00, 0x00, 0x3b, // start ip_hdr + 0xa6, 0xab, 0x40, 0x00, + 0x40, 0x06, 0x96, 0x0f, + 0x7f, 0x00, 0x00, 0x01, + 0x7f, 0x00, 0x00, 0x01, + 0x99, 0x99, 0xc6, 0xcc, // start tcp_hdr + 0xd1, 0xe5, 0xc4, 0x9d, + 0xd4, 0x30, 0xb5, 0xd2, + 0x80, 0x18, 0x01, 0x56, + 0xfe, 0x2f, 0x00, 0x00, + 0x01, 0x01, 0x08, 0x0a, // start data + 0x00, 0x23, 0x75, 0x89, + 0x00, 0x23, 0x63, 0x2d, + 0x71, 0x64, 0x66, 0x73, + 0x64, 0x66, 0x0a + ]; + + // This is an eBPF VM for programs using a virtual metadata buffer, similar + // to the sk_buff that eBPF programs use with tc and in Linux kernel. + // We must provide the offsets at which the pointers to packet data start + // and end must be stored: these are the offsets at which the program will + // load the packet data from the metadata buffer. + let mut vm = rbpf::EbpfVmFixedMbuff::new(Some(prog), 0x40, 0x50).unwrap(); + + // We register a helper function, that can be called by the program, into + // the VM. The `bpf_trace_printf` is only available when we have access to + // the standard library. + #[cfg(feature = "std")] { + vm.register_helper(helpers::BPF_TRACE_PRINTK_IDX, + helpers::bpf_trace_printf).unwrap(); + } + + // This kind of VM takes a reference to the packet data, but does not need + // any reference to the metadata buffer: a fixed buffer is handled + // internally by the VM. + let res = vm.execute_program(packet).unwrap(); + println!("Program returned: {:?} ({:#x})", res, res); +} +``` + +## Building eBPF programs + +Besides passing the raw hexadecimal codes for building eBPF programs, two other +methods are available. + +### Assembler + +The first method consists in using the assembler provided by the crate. + +```rust +extern crate rbpf; +use rbpf::assembler::assemble; + +let prog = assemble("add64 r1, 0x605 + mov64 r2, 0x32 + mov64 r1, r0 + be16 r0 + neg64 r2 + exit").unwrap(); + +#[cfg(feature = "std")] { + println!("{:?}", prog); +} +``` + +The above snippet will produce: + +```rust,ignore +Ok([0x07, 0x01, 0x00, 0x00, 0x05, 0x06, 0x00, 0x00, + 0xb7, 0x02, 0x00, 0x00, 0x32, 0x00, 0x00, 0x00, + 0xbf, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xdc, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, + 0x87, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00]) +``` + +Conversely, a disassembler is also available to dump instruction names from +bytecode in a human-friendly format. + +```rust +extern crate rbpf; +use rbpf::disassembler::disassemble; + +let prog = &[ + 0x07, 0x01, 0x00, 0x00, 0x05, 0x06, 0x00, 0x00, + 0xb7, 0x02, 0x00, 0x00, 0x32, 0x00, 0x00, 0x00, + 0xbf, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xdc, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, + 0x87, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 +]; + +disassemble(prog); +``` + +This will produce the following output: + +```txt +add64 r1, 0x605 +mov64 r2, 0x32 +mov64 r1, r0 +be16 r0 +neg64 r2 +exit +``` + +Please refer to [source code](src/assembler.rs) and [tests](tests/assembler.rs) +for the syntax and the list of instruction names. + +### Building API + +The other way to build programs is to chain commands from the instruction +builder API. It looks less like assembly, maybe more like high-level functions. +What's sure is that the result is more verbose, but if you prefer to build +programs this way, it works just as well. If we take again the same sample as +above, it would be constructed as follows. + +```rust +extern crate rbpf; +use rbpf::insn_builder::*; + +let mut program = BpfCode::new(); +program.add(Source::Imm, Arch::X64).set_dst(1).set_imm(0x605).push() + .mov(Source::Imm, Arch::X64).set_dst(2).set_imm(0x32).push() + .mov(Source::Reg, Arch::X64).set_src(0).set_dst(1).push() + .swap_bytes(Endian::Big).set_dst(0).set_imm(0x10).push() + .negate(Arch::X64).set_dst(2).push() + .exit().push(); +``` + +Again, please refer to [the source and related tests](src/insn_builder.rs) to +get more information and examples on how to use it. + +## Build features + +### `no_std` + +The `rbpf` crate has a Cargo feature named "std" that is enabled by default. To +use `rbpf` in `no_std` environments this feature needs to be disabled. To do +this, you need to modify your dependency on `rbpf` in Cargo.toml to disable the +enabled-by-default features. + +```toml +[dependencies] +rbpf = { version = "1.0", default-features = false } +``` + +Note that when using this crate in `no_std` environments, the `jit` module +isn't available. This is because it depends on functions provided by `libc` +(`libc::posix_memalign()`, `libc::mprotect()`) which aren't available on +`no_std`. + +The `assembler` module is available, albeit with reduced debugging features. It +depends on the `combine` crate providing parser combinators. Under `no_std` +this crate only provides simple parsers which generate less descriptive error +messages. + +## Feedback welcome! + +This is the author's first try at writing Rust code. He learned a lot in the +process, but there remains a feeling that this crate has a kind of C-ish style +in some places instead of the Rusty look the author would like it to have. So +feedback (or PRs) are welcome, including about ways you might see to take +better advantage of Rust features. + +Note that the project expects new commits to be covered by the +[Developer's Certificate of Origin](https://wiki.linuxfoundation.org/dco). +When contributing Pull Requests, please sign off your commits accordingly. + +## Questions / Answers + +### Why implementing an eBPF virtual machine in Rust? + +As of this writing, there is no particular use case for this crate at the best +of the author's knowledge. The author happens to work with BPF on Linux and to +know how uBPF works, and he wanted to learn and experiment with Rust—no more +than that. + +### What are the differences with uBPF? + +Other than the language, obviously? Well, there are some differences: + +* Some constants, such as the maximum length for programs or the length for the + stack, differs between uBPF and rbpf. The latter uses the same values as the + Linux kernel, while uBPF has its own values. + +* When an error occurs while a program is run by uBPF, the function running the + program silently returns the maximum value as an error code, while rbpf + returns Rust type `Error`. + +* The registration of helper functions, that can be called from within an eBPF + program, is not handled in the same way. + +* The distinct structs permitting to run program either on packet data, or with + a metadata buffer (simulated or not) is a specificity of rbpf. + +* As for performance: theoretically the JITted programs are expected to run at + the same speed, while the C interpreter of uBPF should go slightly faster + than rbpf. But this has not been asserted yet. Benchmarking both programs + would be an interesting thing to do. + +### Can I use it with the “classic” BPF (a.k.a cBPF) version? + +No. This crate only works with extended BPF (eBPF) programs. For cBPF programs, +such as used by tcpdump (as of this writing) for example, you may be interested +in the [bpfjit crate](https://crates.io/crates/bpfjit) written by Alexander +Polakov instead. + +### What functionalities are implemented? + +Running and JIT-compiling eBPF programs work. There is also a mechanism to +register user-defined helper functions. The eBPF implementation of the Linux +kernel comes with [some additional +features](https://github.com/iovisor/bcc/blob/master/docs/kernel-versions.md): +a high number of helpers, several kinds of maps, tail calls. + +* Additional helpers should be easy to add, but very few of the existing Linux + helpers have been replicated in rbpf so far. + +* Tail calls (“long jumps” from an eBPF program into another) are not + implemented. This is probably not trivial to design and implement. + +* The interaction with maps is done through the use of specific helpers, so + this should not be difficult to add. The maps themselves can reuse the maps + in the kernel (if on Linux), to communicate with in-kernel eBPF programs for + instance; or they can be handled in user space. Rust has arrays and hashmaps, + so their implementation should be pretty straightforward (and may be added to + rbpf in the future). + +### What about program validation? + +The ”verifier” of this crate is very short and has nothing to do with the +kernel verifier, which means that it accepts programs that may not be safe. On +the other hand, you probably do not run this in a kernel here, so it will not +crash your system. Implementing a verifier similar to the one in the kernel is +not trivial, and we cannot “copy” it since it is under GPL license. + +### What about safety then? + +Rust has a strong emphasis on safety. Yet to have the eBPF VM work, some +`unsafe` blocks of code are used. The VM, taken as an eBPF interpreter, can +return an error but should not crash. Please file an issue otherwise. + +As for the JIT-compiler, it is a different story, since runtime memory checks +are more complicated to implement in assembly. It _will_ crash if your +JIT-compiled program tries to perform unauthorized memory accesses. Usually, it +could be a good idea to test your program with the interpreter first. + +Oh, and if your program has infinite loops, even with the interpreter, you're +on your own. + +## Caveats + +* This crate is **under development** and the API may be subject to change. + +* The JIT compiler produces an unsafe program: memory access are not tested at + runtime (yet). Use with caution. + +* A small number of eBPF instructions have not been implemented yet. This + should not be a problem for the majority of eBPF programs. + +* Beware of turnips. Turnips are disgusting. + +## _To do_ list + +* Implement some traits (`Clone`, `Drop`, `Debug` are good candidates). +* Provide built-in support for user-space array and hash BPF maps. +* Improve safety of JIT-compiled programs with runtime memory checks. +* Add helpers (some of those supported in the kernel, such as checksum update, + could be helpful). +* Improve verifier. Could we find a way to directly support programs compiled + with clang? +* Maybe one day, tail calls? +* JIT-compilers for other architectures? +* … + +## License + +Following the effort of the Rust language project itself in order to ease +integration with other projects, the rbpf crate is distributed under the terms +of both the MIT license and the Apache License (Version 2.0). + +See +[LICENSE-APACHE](https://github.com/qmonnet/rbpf/blob/main/LICENSE-APACHE) +and [LICENSE-MIT](https://github.com/qmonnet/rbpf/blob/main/LICENSE-MIT) for +details. + +## Version +[The last commit](https://github.com/qmonnet/rbpf/commit/fe7021b07b08a43b836743a77796d07ce1f4902e) + + +## Inspired by + +* [uBPF](https://github.com/iovisor/ubpf), a C user-space implementation of an + eBPF virtual machine, with a JIT-compiler and disassembler (and also + including the assembler from the human-readable form of the instructions, + such as in `mov r0, 0x1337`), by Rich Lane for Big Switch Networks (2015) + +* [_Building a simple JIT in + Rust_](https://www.sophiajt.com/building-a-simple-jit-in-rust), + by Sophia Turner (2015) + +* [bpfjit](https://github.com/polachok/bpfjit) (also [on + crates.io](https://crates.io/crates/bpfjit)), a Rust crate exporting the cBPF + JIT compiler from FreeBSD 10 tree to Rust, by Alexander Polakov (2016) + +## Other resources + +* Cilium project documentation about BPF: [_BPF and XDP Reference + Guide_](http://docs.cilium.io/en/latest/bpf/) + +* [Kernel documentation about BPF](https://docs.kernel.org/bpf/) + +* [_Dive into BPF: a list of reading + material_](https://qmonnet.github.io/whirl-offload/2016/09/01/dive-into-bpf), + a blog article listing documentation for BPF and related technologies (2016) + +* [The Rust programming language](https://www.rust-lang.org) diff --git a/kernel/crates/rbpf/clippy.toml b/kernel/crates/rbpf/clippy.toml new file mode 100644 index 000000000..1d4a2968a --- /dev/null +++ b/kernel/crates/rbpf/clippy.toml @@ -0,0 +1 @@ +doc-valid-idents = ["eBPF", "uBPF"] diff --git a/kernel/crates/rbpf/examples/disassemble.rs b/kernel/crates/rbpf/examples/disassemble.rs new file mode 100644 index 000000000..4dbcc277c --- /dev/null +++ b/kernel/crates/rbpf/examples/disassemble.rs @@ -0,0 +1,26 @@ +// SPDX-License-Identifier: (Apache-2.0 OR MIT) +// Copyright 2017 6WIND S.A. + +extern crate rbpf; +use rbpf::disassembler; + +// Simply disassemble a program into human-readable instructions. +fn main() { + let prog = &[ + 0xb7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x79, 0x12, 0x50, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x79, 0x11, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xbf, 0x13, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x07, 0x03, 0x00, 0x00, 0x36, 0x00, 0x00, 0x00, 0x2d, 0x23, 0x12, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x69, 0x12, 0x0c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x55, 0x02, 0x10, 0x00, + 0x08, 0x00, 0x00, 0x00, 0x71, 0x12, 0x17, 0x00, 0x00, 0x00, 0x00, 0x00, 0x55, 0x02, 0x0e, + 0x00, 0x06, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x79, 0x11, 0x22, 0x00, 0x00, 0x00, 0x00, 0x00, 0xbf, + 0x12, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x57, 0x02, 0x00, 0x00, 0xff, 0xff, 0x00, 0x00, + 0x15, 0x02, 0x08, 0x00, 0x99, 0x99, 0x00, 0x00, 0x18, 0x02, 0x00, 0x00, 0x00, 0x00, 0xff, + 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x5f, 0x21, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xb7, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x18, 0x02, 0x00, 0x00, 0x00, + 0x00, 0x99, 0x99, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1d, 0x21, 0x01, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xb7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x95, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, + ]; + disassembler::disassemble(prog); +} diff --git a/kernel/crates/rbpf/examples/helper.rs b/kernel/crates/rbpf/examples/helper.rs new file mode 100644 index 000000000..ace3dfefb --- /dev/null +++ b/kernel/crates/rbpf/examples/helper.rs @@ -0,0 +1,3 @@ +fn main() { + rbpf::helpers::show_helper(); +} diff --git a/kernel/crates/rbpf/examples/load_elf.rs b/kernel/crates/rbpf/examples/load_elf.rs new file mode 100644 index 000000000..9d7941406 --- /dev/null +++ b/kernel/crates/rbpf/examples/load_elf.rs @@ -0,0 +1,115 @@ +// SPDX-License-Identifier: (Apache-2.0 OR MIT) +// Copyright 2016 6WIND S.A. + +#![allow(clippy::unreadable_literal)] + +extern crate elf; +use std::path::PathBuf; + +extern crate rbpf; +use rbpf::helpers; + +// The following example uses an ELF file that has been compiled from the C program available in +// `load_elf__block_a_port.c` in the same directory. +// +// It was compiled with the following command: +// +// ```bash +// clang -O2 -emit-llvm -c load_elf__block_a_port.c -o - | \ +// llc -march=bpf -filetype=obj -o load_elf__block_a_port.o +// ``` +// +// Once compiled, this program can be injected into Linux kernel, with tc for instance. Sadly, we +// need to bring some modifications to the generated bytecode in order to run it: the three +// instructions with opcode 0x61 load data from a packet area as 4-byte words, where we need to +// load it as 8-bytes double words (0x79). The kernel does the same kind of translation before +// running the program, but rbpf does not implement this. +// +// In addition, the offset at which the pointer to the packet data is stored must be changed: since +// we use 8 bytes instead of 4 for the start and end addresses of the data packet, we cannot use +// the offsets produced by clang (0x4c and 0x50), the addresses would overlap. Instead we can use, +// for example, 0x40 and 0x50. +// +// These change were applied with the following script: +// +// ```bash +// xxd load_elf__block_a_port.o | sed ' +// s/6112 5000 0000 0000/7912 5000 0000 0000/ ; +// s/6111 4c00 0000 0000/7911 4000 0000 0000/ ; +// s/6111 2200 0000 0000/7911 2200 0000 0000/' | xxd -r > load_elf__block_a_port.tmp + +// mv load_elf__block_a_port.tmp load_elf__block_a_port.o +// ``` +// +// The eBPF program was placed into the `.classifier` ELF section (see C code above), which means +// that you can retrieve the raw bytecode with `readelf -x .classifier load_elf__block_a_port.o` or +// with `objdump -s -j .classifier load_elf__block_a_port.o`. +// +// Once the bytecode has been edited, we can load the bytecode directly from the ELF object file. + +fn main() { + let filename = "examples/load_elf__block_a_port.elf"; + + let path = PathBuf::from(filename); + let file = match elf::File::open_path(path) { + Ok(f) => f, + Err(e) => panic!("Error: {:?}", e), + }; + + let text_scn = match file.get_section(".classifier") { + Some(s) => s, + None => panic!("Failed to look up .classifier section"), + }; + + let prog = &text_scn.data; + + let packet1 = &mut [ + 0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xfe, 0xdc, 0xba, 0x98, 0x76, 0x54, 0x08, + 0x00, // ethertype + 0x45, 0x00, 0x00, 0x3b, // start ip_hdr + 0xa6, 0xab, 0x40, 0x00, 0x40, 0x06, 0x96, 0x0f, 0x7f, 0x00, 0x00, 0x01, 0x7f, 0x00, 0x00, + 0x01, + // Program matches the next two bytes: 0x9999 returns 0xffffffff, else return 0. + 0x99, 0x99, 0xc6, 0xcc, // start tcp_hdr + 0xd1, 0xe5, 0xc4, 0x9d, 0xd4, 0x30, 0xb5, 0xd2, 0x80, 0x18, 0x01, 0x56, 0xfe, 0x2f, 0x00, + 0x00, 0x01, 0x01, 0x08, 0x0a, // start data + 0x00, 0x23, 0x75, 0x89, 0x00, 0x23, 0x63, 0x2d, 0x71, 0x64, 0x66, 0x73, 0x64, 0x66, 0x0au8, + ]; + + let packet2 = &mut [ + 0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xfe, 0xdc, 0xba, 0x98, 0x76, 0x54, 0x08, + 0x00, // ethertype + 0x45, 0x00, 0x00, 0x3b, // start ip_hdr + 0xa6, 0xab, 0x40, 0x00, 0x40, 0x06, 0x96, 0x0f, 0x7f, 0x00, 0x00, 0x01, 0x7f, 0x00, 0x00, + 0x01, + // Program matches the next two bytes: 0x9999 returns 0xffffffff, else return 0. + 0x98, 0x76, 0xc6, 0xcc, // start tcp_hdr + 0xd1, 0xe5, 0xc4, 0x9d, 0xd4, 0x30, 0xb5, 0xd2, 0x80, 0x18, 0x01, 0x56, 0xfe, 0x2f, 0x00, + 0x00, 0x01, 0x01, 0x08, 0x0a, // start data + 0x00, 0x23, 0x75, 0x89, 0x00, 0x23, 0x63, 0x2d, 0x71, 0x64, 0x66, 0x73, 0x64, 0x66, 0x0au8, + ]; + + let mut vm = rbpf::EbpfVmFixedMbuff::new(Some(prog), 0x40, 0x50).unwrap(); + vm.register_helper(helpers::BPF_TRACE_PRINTK_IDX, helpers::bpf_trace_printf) + .unwrap(); + + let res = vm.execute_program(packet1).unwrap(); + println!("Packet #1, program returned: {res:?} ({res:#x})"); + assert_eq!(res, 0xffffffff); + + #[cfg(not(windows))] + { + vm.jit_compile().unwrap(); + + let res = unsafe { vm.execute_program_jit(packet2).unwrap() }; + println!("Packet #2, program returned: {res:?} ({res:#x})"); + assert_eq!(res, 0); + } + + #[cfg(windows)] + { + let res = vm.execute_program(packet2).unwrap(); + println!("Packet #2, program returned: {:?} ({:#x})", res, res); + assert_eq!(res, 0); + } +} diff --git a/kernel/crates/rbpf/examples/load_elf__block_a_port.c b/kernel/crates/rbpf/examples/load_elf__block_a_port.c new file mode 100644 index 000000000..0a51117ee --- /dev/null +++ b/kernel/crates/rbpf/examples/load_elf__block_a_port.c @@ -0,0 +1,43 @@ +// SPDX-License-Identifier: (APACHE-2.0 OR MIT) +// Copyright 2016 6WIND S.A. + +// Block TCP packets on source or destination port 0x9999. + +#include +#include +#include +#include + +#define ETH_ALEN 6 +#define ETH_P_IP 0x0008 /* htons(0x0800) */ +#define TCP_HDR_LEN 20 + +#define BLOCKED_TCP_PORT 0x9999 + +struct eth_hdr { + unsigned char h_dest[ETH_ALEN]; + unsigned char h_source[ETH_ALEN]; + unsigned short h_proto; +}; + +#define SEC(NAME) __attribute__((section(NAME), used)) +SEC(".classifier") +int handle_ingress(struct __sk_buff *skb) +{ + void *data = (void *)(long)skb->data; + void *data_end = (void *)(long)skb->data_end; + struct eth_hdr *eth = data; + struct iphdr *iph = data + sizeof(*eth); + struct tcphdr *tcp = data + sizeof(*eth) + sizeof(*iph); + + /* single length check */ + if (data + sizeof(*eth) + sizeof(*iph) + sizeof(*tcp) > data_end) + return 0; + if (eth->h_proto != ETH_P_IP) + return 0; + if (iph->protocol != IPPROTO_TCP) + return 0; + if (tcp->source == BLOCKED_TCP_PORT || tcp->dest == BLOCKED_TCP_PORT) + return -1; + return 0; +} diff --git a/kernel/crates/rbpf/examples/rbpf_plugin.rs b/kernel/crates/rbpf/examples/rbpf_plugin.rs new file mode 100644 index 000000000..7a7ace6e5 --- /dev/null +++ b/kernel/crates/rbpf/examples/rbpf_plugin.rs @@ -0,0 +1,126 @@ +// Copyright Microsoft Corporation +// SPDX-License-Identifier: (Apache-2.0 OR MIT) + +// Path: examples/rbpf_plugin.rs +use std::io::Read; + +// Helper function used by https://github.com/Alan-Jowett/bpf_conformance/blob/main/tests/call_unwind_fail.data +fn _unwind(a: u64, _b: u64, _c: u64, _d: u64, _e: u64) -> u64 { + a +} + +// This is a plugin for the bpf_conformance test suite (https://github.com/Alan-Jowett/bpf_conformance) +// It accepts a single argument, the memory contents to pass to the VM. +// It reads the program from stdin. +fn main() { + let mut args: Vec = std::env::args().collect(); + #[allow(unused_mut)] // In no_std the jit variable isn't mutated. + let mut jit: bool = false; + let mut cranelift: bool = false; + let mut program_text = String::new(); + let mut memory_text = String::new(); + + args.remove(0); + + // Memory is always the first argument. + if !args.is_empty() { + memory_text.clone_from(&args[0]); + // Strip whitespace + memory_text.retain(|c| !c.is_whitespace()); + args.remove(0); + } + + // Process the rest of the arguments. + while !args.is_empty() { + match args[0].as_str() { + "--help" => { + println!("Usage: rbpf_plugin [memory] < program"); + return; + } + "--jit" => { + #[cfg(any(windows, not(feature = "std")))] + { + println!("JIT not supported"); + return; + } + #[cfg(all(not(windows), feature = "std"))] + { + jit = true; + } + } + "--cranelift" => { + cranelift = true; + + #[cfg(not(feature = "cranelift"))] + { + let _ = cranelift; + println!("Cranelift is not enabled"); + return; + } + } + "--program" => { + if args.len() < 2 { + println!("Missing argument to --program"); + return; + } + args.remove(0); + if !args.is_empty() { + program_text.clone_from(&args[0]); + args.remove(0); + } + } + _ => panic!("Unknown argument {}", args[0]), + } + args.remove(0); + } + + if program_text.is_empty() { + // Read program text from stdin + std::io::stdin().read_to_string(&mut program_text).unwrap(); + } + + // Strip whitespace + program_text.retain(|c| !c.is_whitespace()); + + // Convert program from hex to bytecode + let bytecode = hex::decode(program_text).unwrap(); + + // Convert memory from hex to bytes + let mut memory: Vec = hex::decode(memory_text).unwrap(); + + // Create rbpf vm + let mut vm = rbpf::EbpfVmRaw::new(Some(&bytecode)).unwrap(); + + // Register the helper function used by call_unwind_fail.data test. + vm.register_helper(5, _unwind).unwrap(); + + let result: u64; + if jit { + #[cfg(any(windows, not(feature = "std")))] + { + println!("JIT not supported"); + return; + } + #[cfg(all(not(windows), feature = "std"))] + { + unsafe { + vm.jit_compile().unwrap(); + result = vm.execute_program_jit(&mut memory).unwrap(); + } + } + } else if cranelift { + #[cfg(not(feature = "cranelift"))] + { + println!("Cranelift is not enabled"); + return; + } + #[cfg(feature = "cranelift")] + { + vm.cranelift_compile().unwrap(); + result = vm.execute_program_cranelift(&mut memory).unwrap(); + } + } else { + result = vm.execute_program(&mut memory).unwrap(); + } + println!("{result:x}"); +} diff --git a/kernel/crates/rbpf/examples/to_json.rs b/kernel/crates/rbpf/examples/to_json.rs new file mode 100644 index 000000000..6b86047da --- /dev/null +++ b/kernel/crates/rbpf/examples/to_json.rs @@ -0,0 +1,74 @@ +// SPDX-License-Identifier: (Apache-2.0 OR MIT) +// Copyright 2017 6WIND S.A. + +#[macro_use] +extern crate json; + +extern crate elf; +use std::path::PathBuf; + +extern crate rbpf; +use rbpf::disassembler; + +// Turn a program into a JSON string. +// +// Relies on `json` crate. +// +// You may copy this function and adapt it according to your needs. For instance, you may want to: +// +// * Remove the "desc" (description) attributes from the output. +// * Print integers as integers, and not as strings containing their hexadecimal representation +// (just replace the relevant `format!()` calls by the commented values. +fn to_json(prog: &[u8]) -> String { + // This call returns a high-level representation of the instructions, with the two parts of + // `LD_DW_IMM` instructions merged, and name and descriptions of the instructions. + // If you prefer to use a lower-level representation, use `ebpf::to_insn_vec()` function + // instead. + let insns = disassembler::to_insn_vec(prog); + let mut json_insns = vec![]; + for insn in insns { + json_insns.push(object!( + "opc" => format!("{:#x}", insn.opc), // => insn.opc, + "dst" => format!("{:#x}", insn.dst), // => insn.dst, + "src" => format!("{:#x}", insn.src), // => insn.src, + "off" => format!("{:#x}", insn.off), // => insn.off, + // Warning: for imm we use a i64 instead of a i32 (to have correct values for + // `lddw` operation. If we print a number in the JSON this is not a problem, the + // internal i64 has the same value with extended sign on 32 most significant bytes. + // If we print the hexadecimal value as a string however, we want to cast as a i32 + // to prevent all other instructions to print spurious `ffffffff` prefix if the + // number is negative. When values takes more than 32 bits with `lddw`, the cast + // has no effect and the complete value is printed anyway. + "imm" => format!("{:#x}", insn.imm as i32), // => insn.imm, + "desc" => insn.desc + )); + } + json::stringify_pretty( + object!( + "size" => json_insns.len(), + "insns" => json_insns + ), + 4, + ) +} + +// Load a program from an object file, and prints it to standard output as a JSON string. +fn main() { + // Let's reuse this file from `load_elf/example`. + let filename = "examples/load_elf__block_a_port.elf"; + + let path = PathBuf::from(filename); + let file = match elf::File::open_path(path) { + Ok(f) => f, + Err(e) => panic!("Error: {:?}", e), + }; + + let text_scn = match file.get_section(".classifier") { + Some(s) => s, + None => panic!("Failed to look up .classifier section"), + }; + + let prog = &text_scn.data; + + println!("{}", to_json(prog)); +} diff --git a/kernel/crates/rbpf/examples/uptime.rs b/kernel/crates/rbpf/examples/uptime.rs new file mode 100644 index 000000000..49b1642ad --- /dev/null +++ b/kernel/crates/rbpf/examples/uptime.rs @@ -0,0 +1,78 @@ +// SPDX-License-Identifier: (Apache-2.0 OR MIT) +// Copyright 2017 6WIND S.A. + +extern crate rbpf; +use rbpf::helpers; + +// The main objectives of this example is to show: +// +// * the use of EbpfVmNoData function, +// * and the use of a helper. +// +// The two eBPF programs are independent and are not related to one another. +fn main() { + let prog1 = &[ + 0xb4, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // mov32 r0, 0 + 0xb4, 0x01, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, // mov32 r1, 2 + 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, // add32 r0, 1 + 0x0c, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // add32 r0, r1 + 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // exit and return r0 + ]; + + // We use helper `bpf_time_getns()`, which is similar to helper `bpf_ktime_getns()` from Linux + // kernel. Hence rbpf::helpers module provides the index of this in-kernel helper as a + // constant, so that we can remain compatible with programs for the kernel. Here we also cast + // it to a u8 so as to use it directly in program instructions. + let hkey = helpers::BPF_KTIME_GETNS_IDX as u8; + let prog2 = &[ + 0xb7, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // mov64 r1, 0 + 0xb7, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // mov64 r1, 0 + 0xb7, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // mov64 r1, 0 + 0xb7, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // mov64 r1, 0 + 0xb7, 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // mov64 r1, 0 + 0x85, 0x00, 0x00, 0x00, hkey, 0x00, 0x00, 0x00, // call helper + 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // exit and return r0 + ]; + + // Create a VM: this one takes no data. Load prog1 in it. + let mut vm = rbpf::EbpfVmNoData::new(Some(prog1)).unwrap(); + // Execute prog1. + assert_eq!(vm.execute_program().unwrap(), 0x3); + + // As struct EbpfVmNoData does not takes any memory area, its return value is mostly + // deterministic. So we know prog1 will always return 3. There is an exception: when it uses + // helpers, the latter may have non-deterministic values, and all calls may not return the same + // value. + // + // In the following example we use a helper to get the elapsed time since boot time: we + // reimplement uptime in eBPF, in Rust. Because why not. + + vm.set_program(prog2).unwrap(); + vm.register_helper(helpers::BPF_KTIME_GETNS_IDX, helpers::bpf_time_getns) + .unwrap(); + + let time; + + #[cfg(all(not(windows), feature = "std"))] + { + vm.jit_compile().unwrap(); + + time = unsafe { vm.execute_program_jit().unwrap() }; + } + + #[cfg(any(windows, not(feature = "std")))] + { + time = vm.execute_program().unwrap(); + } + + let days = time / 10u64.pow(9) / 60 / 60 / 24; + let hours = (time / 10u64.pow(9) / 60 / 60) % 24; + let minutes = (time / 10u64.pow(9) / 60) % 60; + let seconds = (time / 10u64.pow(9)) % 60; + let nanosec = time % 10u64.pow(9); + + println!( + "Uptime: {:#x} ns == {} days {:02}:{:02}:{:02}, {} ns", + time, days, hours, minutes, seconds, nanosec + ); +} diff --git a/kernel/crates/rbpf/mk/appveyor.bat b/kernel/crates/rbpf/mk/appveyor.bat new file mode 100644 index 000000000..06f61d39f --- /dev/null +++ b/kernel/crates/rbpf/mk/appveyor.bat @@ -0,0 +1,72 @@ +echo on +SetLocal EnableDelayedExpansion + +REM This is the recommended way to choose the toolchain version, according to +REM Appveyor's documentation. +SET PATH=C:\Program Files (x86)\MSBuild\%TOOLCHAIN_VERSION%\Bin;%PATH% + +set VCVARSALL="C:\Program Files (x86)\Microsoft Visual Studio %TOOLCHAIN_VERSION%\VC\vcvarsall.bat" + +if [%Platform%] NEQ [x64] goto win32 +set TARGET_ARCH=x86_64 +set TARGET_PROGRAM_FILES=%ProgramFiles% +call %VCVARSALL% amd64 +if %ERRORLEVEL% NEQ 0 exit 1 +goto download + +:win32 +echo on +if [%Platform%] NEQ [Win32] exit 1 +set TARGET_ARCH=i686 +set TARGET_PROGRAM_FILES=%ProgramFiles(x86)% +call %VCVARSALL% amd64_x86 +if %ERRORLEVEL% NEQ 0 exit 1 +goto download + +:download +REM vcvarsall turns echo off +echo on + +mkdir windows_build_tools +mkdir windows_build_tools\ +echo Downloading Yasm... +powershell -Command "(New-Object Net.WebClient).DownloadFile('http://www.tortall.net/projects/yasm/releases/yasm-1.3.0-win64.exe', 'windows_build_tools\yasm.exe')" +if %ERRORLEVEL% NEQ 0 ( + echo ...downloading Yasm failed. + exit 1 +) + +set RUST_URL=https://static.rust-lang.org/dist/rust-%RUST%-%TARGET_ARCH%-pc-windows-msvc.msi +echo Downloading %RUST_URL%... +mkdir build +powershell -Command "(New-Object Net.WebClient).DownloadFile('%RUST_URL%', 'build\rust-%RUST%-%TARGET_ARCH%-pc-windows-msvc.msi')" +if %ERRORLEVEL% NEQ 0 ( + echo ...downloading Rust failed. + exit 1 +) + +start /wait msiexec /i build\rust-%RUST%-%TARGET_ARCH%-pc-windows-msvc.msi INSTALLDIR="%TARGET_PROGRAM_FILES%\Rust %RUST%" /quiet /qn /norestart +if %ERRORLEVEL% NEQ 0 exit 1 + +set PATH="%TARGET_PROGRAM_FILES%\Rust %RUST%\bin";%cd%\windows_build_tools;%PATH% + +if [%Configuration%] == [Release] set CARGO_MODE=--release + +set + +link /? +cl /? +rustc --version +cargo --version + +cargo test --all-features -vv %CARGO_MODE% +if %ERRORLEVEL% NEQ 0 exit 1 + +REM Verify that `cargo build`, independent from `cargo test`, works; i.e. +REM verify that non-test builds aren't trying to use test-only features. +cargo build -vv %CARGO_MODE% +if %ERRORLEVEL% NEQ 0 exit 1 + +REM Verify that we can build with all features +cargo build --all-features -vv %CARGO_MODE% +if %ERRORLEVEL% NEQ 0 exit 1 diff --git a/kernel/crates/rbpf/rustfmt.toml b/kernel/crates/rbpf/rustfmt.toml new file mode 100644 index 000000000..8a0a3841e --- /dev/null +++ b/kernel/crates/rbpf/rustfmt.toml @@ -0,0 +1,3 @@ +group_imports="StdExternalCrate" +reorder_imports=true +imports_granularity="Crate" \ No newline at end of file diff --git a/kernel/crates/rbpf/src/asm_parser.rs b/kernel/crates/rbpf/src/asm_parser.rs new file mode 100644 index 000000000..41969218a --- /dev/null +++ b/kernel/crates/rbpf/src/asm_parser.rs @@ -0,0 +1,642 @@ +// SPDX-License-Identifier: (Apache-2.0 OR MIT) +// Copyright 2017 Rich Lane + +// Rust-doc comments were left in the module, but it is no longer publicly exposed from the root +// file of the crate. Do not expect to find those comments in the documentation of the crate. + +//! This module parses eBPF assembly language source code. + +use alloc::{ + string::{String, ToString}, + vec::Vec, +}; + +#[cfg(feature = "std")] +use combine::EasyParser; +use combine::{ + attempt, between, eof, many, many1, one_of, optional, + parser::char::{alpha_num, char, digit, hex_digit, spaces, string}, + sep_by, + stream::position::{self}, + ParseError, Parser, Stream, +}; + +/// Operand of an instruction. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Operand { + /// Register number. + Register(i64), + /// Jump offset or immediate. + Integer(i64), + /// Register number and offset. + Memory(i64, i64), + /// Used for pattern matching. + Nil, +} + +/// Parsed instruction. +#[derive(Debug, PartialEq, Eq)] +pub struct Instruction { + /// Instruction name. + pub name: String, + /// Operands. + pub operands: Vec, +} + +fn ident() -> impl Parser +where + I: Stream, + I::Error: ParseError, +{ + many1(alpha_num()) +} + +fn integer() -> impl Parser +where + I: Stream, + I::Error: ParseError, +{ + let sign = optional(one_of("-+".chars())).map(|x| match x { + Some('-') => -1, + _ => 1, + }); + let hex = string("0x") + .with(many1(hex_digit())) + .map(|x: String| u64::from_str_radix(&x, 16).unwrap() as i64); + let dec = many1(digit()).map(|x: String| x.parse::().unwrap()); + (sign, attempt(hex).or(dec)).map(|(s, x)| s * x) +} + +fn register() -> impl Parser +where + I: Stream, + I::Error: ParseError, +{ + char('r') + .with(many1(digit())) + .map(|x: String| x.parse::().unwrap()) +} + +fn operand() -> impl Parser +where + I: Stream, + I::Error: ParseError, +{ + let register_operand = register().map(Operand::Register); + let immediate = integer().map(Operand::Integer); + let memory = between(char('['), char(']'), (register(), optional(integer()))) + .map(|t| Operand::Memory(t.0, t.1.unwrap_or(0))); + register_operand.or(immediate).or(memory) +} + +fn instruction() -> impl Parser +where + I: Stream, + I::Error: ParseError, +{ + let operands = sep_by(operand(), char(',').skip(spaces())); + (ident().skip(spaces()), operands, spaces()).map(|t| Instruction { + name: t.0, + operands: t.1, + }) +} + +/// Parse a string into a list of instructions. +/// +/// The instructions are not validated and may have invalid names and operand types. +pub fn parse(input: &str) -> Result, String> { + let mut with = spaces().with(many(instruction()).skip(eof())); + + #[cfg(feature = "std")] + { + match with.easy_parse(position::Stream::new(input)) { + Ok((insts, _)) => Ok(insts), + Err(err) => Err(err.to_string()), + } + } + #[cfg(not(feature = "std"))] + { + match with.parse(position::Stream::new(input)) { + Ok((insts, _)) => Ok(insts), + Err(err) => Err(err.to_string()), + } + } +} + +#[cfg(test)] +mod tests { + use alloc::{string::ToString, vec}; + + use combine::Parser; + + use super::{ident, instruction, integer, operand, parse, register, Instruction, Operand}; + + // Unit tests for the different kinds of parsers. + + #[test] + fn test_ident() { + assert_eq!(ident().parse("nop"), Ok(("nop".to_string(), ""))); + assert_eq!(ident().parse("add32"), Ok(("add32".to_string(), ""))); + assert_eq!(ident().parse("add32*"), Ok(("add32".to_string(), "*"))); + } + + #[test] + fn test_integer() { + assert_eq!(integer().parse("0"), Ok((0, ""))); + assert_eq!(integer().parse("42"), Ok((42, ""))); + assert_eq!(integer().parse("+42"), Ok((42, ""))); + assert_eq!(integer().parse("-42"), Ok((-42, ""))); + assert_eq!(integer().parse("0x0"), Ok((0, ""))); + assert_eq!( + integer().parse("0x123456789abcdef0"), + Ok((0x123456789abcdef0, "")) + ); + assert_eq!(integer().parse("-0x1f"), Ok((-31, ""))); + } + + #[test] + fn test_register() { + assert_eq!(register().parse("r0"), Ok((0, ""))); + assert_eq!(register().parse("r15"), Ok((15, ""))); + } + + #[test] + fn test_operand() { + assert_eq!(operand().parse("r0"), Ok((Operand::Register(0), ""))); + assert_eq!(operand().parse("r15"), Ok((Operand::Register(15), ""))); + assert_eq!(operand().parse("0"), Ok((Operand::Integer(0), ""))); + assert_eq!(operand().parse("42"), Ok((Operand::Integer(42), ""))); + assert_eq!(operand().parse("[r1]"), Ok((Operand::Memory(1, 0), ""))); + assert_eq!(operand().parse("[r3+5]"), Ok((Operand::Memory(3, 5), ""))); + assert_eq!( + operand().parse("[r3+0x1f]"), + Ok((Operand::Memory(3, 31), "")) + ); + assert_eq!( + operand().parse("[r3-0x1f]"), + Ok((Operand::Memory(3, -31), "")) + ); + } + + #[test] + fn test_instruction() { + assert_eq!( + instruction().parse("exit"), + Ok(( + Instruction { + name: "exit".to_string(), + operands: vec![], + }, + "" + )) + ); + + assert_eq!( + instruction().parse("call 2"), + Ok(( + Instruction { + name: "call".to_string(), + operands: vec![Operand::Integer(2)], + }, + "" + )) + ); + + assert_eq!( + instruction().parse("addi r1, 2"), + Ok(( + Instruction { + name: "addi".to_string(), + operands: vec![Operand::Register(1), Operand::Integer(2)], + }, + "" + )) + ); + + assert_eq!( + instruction().parse("ldxb r2, [r1+12]"), + Ok(( + Instruction { + name: "ldxb".to_string(), + operands: vec![Operand::Register(2), Operand::Memory(1, 12)], + }, + "" + )) + ); + + assert_eq!( + instruction().parse("lsh r3, 0x8"), + Ok(( + Instruction { + name: "lsh".to_string(), + operands: vec![Operand::Register(3), Operand::Integer(8)], + }, + "" + )) + ); + + assert_eq!( + instruction().parse("jne r3, 0x8, +37"), + Ok(( + Instruction { + name: "jne".to_string(), + operands: vec![ + Operand::Register(3), + Operand::Integer(8), + Operand::Integer(37) + ], + }, + "" + )) + ); + + // Whitespace between operands is optional. + assert_eq!( + instruction().parse("jne r3,0x8,+37"), + Ok(( + Instruction { + name: "jne".to_string(), + operands: vec![ + Operand::Register(3), + Operand::Integer(8), + Operand::Integer(37) + ], + }, + "" + )) + ); + } + + // Other unit tests: try to parse various set of instructions. + + #[test] + fn test_empty() { + assert_eq!(parse(""), Ok(vec![])); + } + + #[test] + fn test_exit() { + // No operands. + assert_eq!( + parse("exit"), + Ok(vec![Instruction { + name: "exit".to_string(), + operands: vec![], + }]) + ); + } + + #[test] + fn test_lsh() { + // Register and immediate operands. + assert_eq!( + parse("lsh r3, 0x20"), + Ok(vec![Instruction { + name: "lsh".to_string(), + operands: vec![Operand::Register(3), Operand::Integer(0x20)], + }]) + ); + } + + #[test] + fn test_ja() { + // Jump offset operand. + assert_eq!( + parse("ja +1"), + Ok(vec![Instruction { + name: "ja".to_string(), + operands: vec![Operand::Integer(1)], + }]) + ); + } + + #[test] + fn test_ldxh() { + // Register and memory operands. + assert_eq!( + parse("ldxh r4, [r1+12]"), + Ok(vec![Instruction { + name: "ldxh".to_string(), + operands: vec![Operand::Register(4), Operand::Memory(1, 12)], + }]) + ); + } + + #[test] + fn test_tcp_sack() { + // Sample program from ubpf. + // We could technically indent the instructions since the parser support white spaces at + // the beginning, but there is another test for that. + let src = "\ +ldxb r2, [r1+12] +ldxb r3, [r1+13] +lsh r3, 0x8 +or r3, r2 +mov r0, 0x0 +jne r3, 0x8, +37 +ldxb r2, [r1+23] +jne r2, 0x6, +35 +ldxb r2, [r1+14] +add r1, 0xe +and r2, 0xf +lsh r2, 0x2 +add r1, r2 +mov r0, 0x0 +ldxh r4, [r1+12] +add r1, 0x14 +rsh r4, 0x2 +and r4, 0x3c +mov r2, r4 +add r2, 0xffffffec +mov r5, 0x15 +mov r3, 0x0 +jgt r5, r4, +20 +mov r5, r3 +lsh r5, 0x20 +arsh r5, 0x20 +mov r4, r1 +add r4, r5 +ldxb r5, [r4] +jeq r5, 0x1, +4 +jeq r5, 0x0, +12 +mov r6, r3 +jeq r5, 0x5, +9 +ja +2 +add r3, 0x1 +mov r6, r3 +ldxb r3, [r4+1] +add r3, r6 +lsh r3, 0x20 +arsh r3, 0x20 +jsgt r2, r3, -18 +ja +1 +mov r0, 0x1 +exit +"; + + assert_eq!( + parse(src), + Ok(vec![ + Instruction { + name: "ldxb".to_string(), + operands: vec![Operand::Register(2), Operand::Memory(1, 12)], + }, + Instruction { + name: "ldxb".to_string(), + operands: vec![Operand::Register(3), Operand::Memory(1, 13)], + }, + Instruction { + name: "lsh".to_string(), + operands: vec![Operand::Register(3), Operand::Integer(8)], + }, + Instruction { + name: "or".to_string(), + operands: vec![Operand::Register(3), Operand::Register(2)], + }, + Instruction { + name: "mov".to_string(), + operands: vec![Operand::Register(0), Operand::Integer(0)], + }, + Instruction { + name: "jne".to_string(), + operands: vec![ + Operand::Register(3), + Operand::Integer(8), + Operand::Integer(37) + ], + }, + Instruction { + name: "ldxb".to_string(), + operands: vec![Operand::Register(2), Operand::Memory(1, 23)], + }, + Instruction { + name: "jne".to_string(), + operands: vec![ + Operand::Register(2), + Operand::Integer(6), + Operand::Integer(35) + ], + }, + Instruction { + name: "ldxb".to_string(), + operands: vec![Operand::Register(2), Operand::Memory(1, 14)], + }, + Instruction { + name: "add".to_string(), + operands: vec![Operand::Register(1), Operand::Integer(14)], + }, + Instruction { + name: "and".to_string(), + operands: vec![Operand::Register(2), Operand::Integer(15)], + }, + Instruction { + name: "lsh".to_string(), + operands: vec![Operand::Register(2), Operand::Integer(2)], + }, + Instruction { + name: "add".to_string(), + operands: vec![Operand::Register(1), Operand::Register(2)], + }, + Instruction { + name: "mov".to_string(), + operands: vec![Operand::Register(0), Operand::Integer(0)], + }, + Instruction { + name: "ldxh".to_string(), + operands: vec![Operand::Register(4), Operand::Memory(1, 12)], + }, + Instruction { + name: "add".to_string(), + operands: vec![Operand::Register(1), Operand::Integer(20)], + }, + Instruction { + name: "rsh".to_string(), + operands: vec![Operand::Register(4), Operand::Integer(2)], + }, + Instruction { + name: "and".to_string(), + operands: vec![Operand::Register(4), Operand::Integer(60)], + }, + Instruction { + name: "mov".to_string(), + operands: vec![Operand::Register(2), Operand::Register(4)], + }, + Instruction { + name: "add".to_string(), + operands: vec![Operand::Register(2), Operand::Integer(4294967276)], + }, + Instruction { + name: "mov".to_string(), + operands: vec![Operand::Register(5), Operand::Integer(21)], + }, + Instruction { + name: "mov".to_string(), + operands: vec![Operand::Register(3), Operand::Integer(0)], + }, + Instruction { + name: "jgt".to_string(), + operands: vec![ + Operand::Register(5), + Operand::Register(4), + Operand::Integer(20) + ], + }, + Instruction { + name: "mov".to_string(), + operands: vec![Operand::Register(5), Operand::Register(3)], + }, + Instruction { + name: "lsh".to_string(), + operands: vec![Operand::Register(5), Operand::Integer(32)], + }, + Instruction { + name: "arsh".to_string(), + operands: vec![Operand::Register(5), Operand::Integer(32)], + }, + Instruction { + name: "mov".to_string(), + operands: vec![Operand::Register(4), Operand::Register(1)], + }, + Instruction { + name: "add".to_string(), + operands: vec![Operand::Register(4), Operand::Register(5)], + }, + Instruction { + name: "ldxb".to_string(), + operands: vec![Operand::Register(5), Operand::Memory(4, 0)], + }, + Instruction { + name: "jeq".to_string(), + operands: vec![ + Operand::Register(5), + Operand::Integer(1), + Operand::Integer(4) + ], + }, + Instruction { + name: "jeq".to_string(), + operands: vec![ + Operand::Register(5), + Operand::Integer(0), + Operand::Integer(12) + ], + }, + Instruction { + name: "mov".to_string(), + operands: vec![Operand::Register(6), Operand::Register(3)], + }, + Instruction { + name: "jeq".to_string(), + operands: vec![ + Operand::Register(5), + Operand::Integer(5), + Operand::Integer(9) + ], + }, + Instruction { + name: "ja".to_string(), + operands: vec![Operand::Integer(2)], + }, + Instruction { + name: "add".to_string(), + operands: vec![Operand::Register(3), Operand::Integer(1)], + }, + Instruction { + name: "mov".to_string(), + operands: vec![Operand::Register(6), Operand::Register(3)], + }, + Instruction { + name: "ldxb".to_string(), + operands: vec![Operand::Register(3), Operand::Memory(4, 1)], + }, + Instruction { + name: "add".to_string(), + operands: vec![Operand::Register(3), Operand::Register(6)], + }, + Instruction { + name: "lsh".to_string(), + operands: vec![Operand::Register(3), Operand::Integer(32)], + }, + Instruction { + name: "arsh".to_string(), + operands: vec![Operand::Register(3), Operand::Integer(32)], + }, + Instruction { + name: "jsgt".to_string(), + operands: vec![ + Operand::Register(2), + Operand::Register(3), + Operand::Integer(-18) + ], + }, + Instruction { + name: "ja".to_string(), + operands: vec![Operand::Integer(1)], + }, + Instruction { + name: "mov".to_string(), + operands: vec![Operand::Register(0), Operand::Integer(1)], + }, + Instruction { + name: "exit".to_string(), + operands: vec![], + } + ]) + ); + } + + /// When running without `std` the `EasyParser` provided by `combine` + /// cannot be used. Because of this we need to use the `Parser` and the + /// error messages are different. + #[test] + fn test_error_eof() { + let expected_error; + #[cfg(feature = "std")] + { + expected_error = Err( + "Parse error at line: 1, column: 6\nUnexpected end of input\nExpected digit\n" + .to_string(), + ); + } + #[cfg(not(feature = "std"))] + { + expected_error = Err("unexpected parse".to_string()); + } + // Unexpected end of input in a register name. + assert_eq!(parse("lsh r"), expected_error); + } + + /// When running without `std` the `EasyParser` provided by `combine` + /// cannot be used. Because of this we need to use the `Parser` and the + /// error messages are different. + #[test] + fn test_error_unexpected_character() { + let expected_error; + #[cfg(feature = "std")] + { + expected_error = Err( + "Parse error at line: 2, column: 1\nUnexpected `^`\nExpected letter or digit, whitespaces, `r`, `-`, `+`, `[` or end of input\n".to_string() + ); + } + #[cfg(not(feature = "std"))] + { + expected_error = Err("unexpected parse".to_string()); + } + // Unexpected character at end of input. + assert_eq!(parse("exit\n^"), expected_error); + } + + #[test] + fn test_initial_whitespace() { + assert_eq!( + parse( + " + exit" + ), + Ok(vec![Instruction { + name: "exit".to_string(), + operands: vec![], + }]) + ); + } +} diff --git a/kernel/crates/rbpf/src/assembler.rs b/kernel/crates/rbpf/src/assembler.rs new file mode 100644 index 000000000..bf452b1e9 --- /dev/null +++ b/kernel/crates/rbpf/src/assembler.rs @@ -0,0 +1,277 @@ +// SPDX-License-Identifier: (Apache-2.0 OR MIT) +// Copyright 2017 Rich Lane + +//! This module translates eBPF assembly language to binary. + +use alloc::{ + collections::BTreeMap, + format, + string::{String, ToString}, + vec, + vec::Vec, +}; + +use self::InstructionType::{ + AluBinary, AluUnary, Call, Endian, JumpConditional, JumpUnconditional, LoadAbs, LoadImm, + LoadInd, LoadReg, NoOperand, StoreImm, StoreReg, +}; +use crate::{ + asm_parser::{ + parse, Instruction, Operand, + Operand::{Integer, Memory, Nil, Register}, + }, + ebpf::{self, Insn}, +}; + +#[derive(Clone, Copy, Debug, PartialEq)] +enum InstructionType { + AluBinary, + AluUnary, + LoadImm, + LoadAbs, + LoadInd, + LoadReg, + StoreImm, + StoreReg, + JumpUnconditional, + JumpConditional, + Call, + Endian(i64), + NoOperand, +} + +fn make_instruction_map() -> BTreeMap { + let mut result = BTreeMap::new(); + + let alu_binary_ops = [ + ("add", ebpf::BPF_ADD), + ("sub", ebpf::BPF_SUB), + ("mul", ebpf::BPF_MUL), + ("div", ebpf::BPF_DIV), + ("or", ebpf::BPF_OR), + ("and", ebpf::BPF_AND), + ("lsh", ebpf::BPF_LSH), + ("rsh", ebpf::BPF_RSH), + ("mod", ebpf::BPF_MOD), + ("xor", ebpf::BPF_XOR), + ("mov", ebpf::BPF_MOV), + ("arsh", ebpf::BPF_ARSH), + ]; + + let mem_sizes = [ + ("w", ebpf::BPF_W), + ("h", ebpf::BPF_H), + ("b", ebpf::BPF_B), + ("dw", ebpf::BPF_DW), + ]; + + let jump_conditions = [ + ("jeq", ebpf::BPF_JEQ), + ("jgt", ebpf::BPF_JGT), + ("jge", ebpf::BPF_JGE), + ("jlt", ebpf::BPF_JLT), + ("jle", ebpf::BPF_JLE), + ("jset", ebpf::BPF_JSET), + ("jne", ebpf::BPF_JNE), + ("jsgt", ebpf::BPF_JSGT), + ("jsge", ebpf::BPF_JSGE), + ("jslt", ebpf::BPF_JSLT), + ("jsle", ebpf::BPF_JSLE), + ]; + + { + let mut entry = |name: &str, inst_type: InstructionType, opc: u8| { + result.insert(name.to_string(), (inst_type, opc)) + }; + + // Miscellaneous. + entry("exit", NoOperand, ebpf::EXIT); + entry("ja", JumpUnconditional, ebpf::JA); + entry("call", Call, ebpf::CALL); + entry("lddw", LoadImm, ebpf::LD_DW_IMM); + + // AluUnary. + entry("neg", AluUnary, ebpf::NEG64); + entry("neg32", AluUnary, ebpf::NEG32); + entry("neg64", AluUnary, ebpf::NEG64); + + // AluBinary. + for &(name, opc) in &alu_binary_ops { + entry(name, AluBinary, ebpf::BPF_ALU64 | opc); + entry(&format!("{name}32"), AluBinary, ebpf::BPF_ALU | opc); + entry(&format!("{name}64"), AluBinary, ebpf::BPF_ALU64 | opc); + } + + // LoadAbs, LoadInd, LoadReg, StoreImm, and StoreReg. + for &(suffix, size) in &mem_sizes { + entry( + &format!("ldabs{suffix}"), + LoadAbs, + ebpf::BPF_ABS | ebpf::BPF_LD | size, + ); + entry( + &format!("ldind{suffix}"), + LoadInd, + ebpf::BPF_IND | ebpf::BPF_LD | size, + ); + entry( + &format!("ldx{suffix}"), + LoadReg, + ebpf::BPF_MEM | ebpf::BPF_LDX | size, + ); + entry( + &format!("st{suffix}"), + StoreImm, + ebpf::BPF_MEM | ebpf::BPF_ST | size, + ); + entry( + &format!("stx{suffix}"), + StoreReg, + ebpf::BPF_MEM | ebpf::BPF_STX | size, + ); + } + + // JumpConditional. + for &(name, condition) in &jump_conditions { + entry(name, JumpConditional, ebpf::BPF_JMP | condition); + entry( + &format!("{name}32"), + JumpConditional, + ebpf::BPF_JMP32 | condition, + ); + } + + // Endian. + for &size in &[16, 32, 64] { + entry(&format!("be{size}"), Endian(size), ebpf::BE); + entry(&format!("le{size}"), Endian(size), ebpf::LE); + } + } + + result +} + +fn insn(opc: u8, dst: i64, src: i64, off: i64, imm: i64) -> Result { + if !(0..16).contains(&dst) { + return Err(format!("Invalid destination register {dst}")); + } + if dst < 0 || src >= 16 { + return Err(format!("Invalid source register {src}")); + } + if !(-32768..32768).contains(&off) { + return Err(format!("Invalid offset {off}")); + } + if !(-2147483648..2147483648).contains(&imm) { + return Err(format!("Invalid immediate {imm}")); + } + Ok(Insn { + opc, + dst: dst as u8, + src: src as u8, + off: off as i16, + imm: imm as i32, + }) +} + +// TODO Use slice patterns when available and remove this function. +fn operands_tuple(operands: &[Operand]) -> Result<(Operand, Operand, Operand), String> { + match operands.len() { + 0 => Ok((Nil, Nil, Nil)), + 1 => Ok((operands[0], Nil, Nil)), + 2 => Ok((operands[0], operands[1], Nil)), + 3 => Ok((operands[0], operands[1], operands[2])), + _ => Err("Too many operands".to_string()), + } +} + +fn encode(inst_type: InstructionType, opc: u8, operands: &[Operand]) -> Result { + let (a, b, c) = (operands_tuple(operands))?; + match (inst_type, a, b, c) { + (AluBinary, Register(dst), Register(src), Nil) => insn(opc | ebpf::BPF_X, dst, src, 0, 0), + (AluBinary, Register(dst), Integer(imm), Nil) => insn(opc | ebpf::BPF_K, dst, 0, 0, imm), + (AluUnary, Register(dst), Nil, Nil) => insn(opc, dst, 0, 0, 0), + (LoadAbs, Integer(imm), Nil, Nil) => insn(opc, 0, 0, 0, imm), + (LoadInd, Register(src), Integer(imm), Nil) => insn(opc, 0, src, 0, imm), + (LoadReg, Register(dst), Memory(src, off), Nil) + | (StoreReg, Memory(dst, off), Register(src), Nil) => insn(opc, dst, src, off, 0), + (StoreImm, Memory(dst, off), Integer(imm), Nil) => insn(opc, dst, 0, off, imm), + (NoOperand, Nil, Nil, Nil) => insn(opc, 0, 0, 0, 0), + (JumpUnconditional, Integer(off), Nil, Nil) => insn(opc, 0, 0, off, 0), + (JumpConditional, Register(dst), Register(src), Integer(off)) => { + insn(opc | ebpf::BPF_X, dst, src, off, 0) + } + (JumpConditional, Register(dst), Integer(imm), Integer(off)) => { + insn(opc | ebpf::BPF_K, dst, 0, off, imm) + } + (Call, Integer(imm), Nil, Nil) => insn(opc, 0, 0, 0, imm), + (Endian(size), Register(dst), Nil, Nil) => insn(opc, dst, 0, 0, size), + (LoadImm, Register(dst), Integer(imm), Nil) => insn(opc, dst, 0, 0, (imm << 32) >> 32), + _ => Err(format!("Unexpected operands: {operands:?}")), + } +} + +fn assemble_internal(parsed: &[Instruction]) -> Result, String> { + let instruction_map = make_instruction_map(); + let mut result: Vec = vec![]; + for instruction in parsed { + let name = instruction.name.as_str(); + match instruction_map.get(name) { + Some(&(inst_type, opc)) => { + match encode(inst_type, opc, &instruction.operands) { + Ok(insn) => result.push(insn), + Err(msg) => return Err(format!("Failed to encode {name}: {msg}")), + } + // Special case for lddw. + if let LoadImm = inst_type { + if let Integer(imm) = instruction.operands[1] { + result.push(insn(0, 0, 0, 0, imm >> 32).unwrap()); + } + } + } + None => return Err(format!("Invalid instruction {name:?}")), + } + } + Ok(result) +} + +/// Parse assembly source and translate to binary. +/// +/// # Examples +/// +/// ``` +/// use rbpf::assembler::assemble; +/// let prog = assemble("add64 r1, 0x605 +/// mov64 r2, 0x32 +/// mov64 r1, r0 +/// be16 r0 +/// neg64 r2 +/// exit"); +/// println!("{:?}", prog); +/// # assert_eq!(prog, +/// # Ok(vec![0x07, 0x01, 0x00, 0x00, 0x05, 0x06, 0x00, 0x00, +/// # 0xb7, 0x02, 0x00, 0x00, 0x32, 0x00, 0x00, 0x00, +/// # 0xbf, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +/// # 0xdc, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, +/// # 0x87, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +/// # 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00])); +/// ``` +/// +/// This will produce the following output: +/// +/// ```test +/// Ok([0x07, 0x01, 0x00, 0x00, 0x05, 0x06, 0x00, 0x00, +/// 0xb7, 0x02, 0x00, 0x00, 0x32, 0x00, 0x00, 0x00, +/// 0xbf, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +/// 0xdc, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, +/// 0x87, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +/// 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00]) +/// ``` +pub fn assemble(src: &str) -> Result, String> { + let parsed = (parse(src))?; + let insns = (assemble_internal(&parsed))?; + let mut result: Vec = vec![]; + for insn in insns { + result.extend_from_slice(&insn.to_array()); + } + Ok(result) +} diff --git a/kernel/crates/rbpf/src/cranelift.rs b/kernel/crates/rbpf/src/cranelift.rs new file mode 100644 index 000000000..10031e5bd --- /dev/null +++ b/kernel/crates/rbpf/src/cranelift.rs @@ -0,0 +1,1230 @@ +// SPDX-License-Identifier: (Apache-2.0 OR MIT) + +use alloc::{collections::BTreeMap, format, vec, vec::Vec}; +use core::{mem, mem::ManuallyDrop}; +use std::io::ErrorKind; + +use cranelift_codegen::{ + entity::EntityRef, + ir::{ + condcodes::IntCC, + types::{I16, I32, I64, I8}, + AbiParam, Block, Endianness, FuncRef, Function, InstBuilder, MemFlags, Signature, + SourceLoc, StackSlotData, StackSlotKind, TrapCode, Type, UserFuncName, Value, + }, + isa::OwnedTargetIsa, + settings::{self, Configurable}, +}; +use cranelift_frontend::{FunctionBuilder, FunctionBuilderContext, Variable}; +use cranelift_jit::{JITBuilder, JITModule}; +use cranelift_module::{FuncId, Linkage, Module}; + +use super::{Error, HashMap, HashSet}; +use crate::ebpf::{ + self, Insn, BPF_ALU_OP_MASK, BPF_IND, BPF_JEQ, BPF_JGE, BPF_JGT, BPF_JLE, BPF_JLT, BPF_JMP32, + BPF_JNE, BPF_JSET, BPF_JSGE, BPF_JSGT, BPF_JSLE, BPF_JSLT, BPF_X, STACK_SIZE, +}; + +pub type JittedFunction = extern "C" fn( + *mut u8, // mem_ptr + usize, // mem_len + *mut u8, // mbuff_ptr + usize, // mbuff_len +) -> u64; + +pub(crate) struct CraneliftCompiler { + isa: OwnedTargetIsa, + module: JITModule, + + helpers: HashMap, + helper_func_refs: HashMap, + + /// List of blocks corresponding to each instruction. + /// We only store the first instruction that observes a new block + insn_blocks: BTreeMap, + /// Map of block targets for each jump/branching instruction. + insn_targets: BTreeMap, + filled_blocks: HashSet, + + /// Map of register numbers to Cranelift variables. + registers: [Variable; 11], + /// Other usefull variables used throughout the program. + mem_start: Variable, + mem_end: Variable, + mbuf_start: Variable, + mbuf_end: Variable, + stack_start: Variable, + stack_end: Variable, +} + +impl CraneliftCompiler { + pub(crate) fn new(helpers: HashMap) -> Self { + let mut flag_builder = settings::builder(); + + flag_builder.set("opt_level", "speed").unwrap(); + + // Enable stack probes + flag_builder.enable("enable_probestack").unwrap(); + flag_builder.set("probestack_strategy", "inline").unwrap(); + + let isa_builder = cranelift_native::builder().unwrap_or_else(|msg| { + panic!("host machine is not supported: {}", msg); + }); + let isa = isa_builder + .finish(settings::Flags::new(flag_builder)) + .unwrap(); + + let mut jit_builder = + JITBuilder::with_isa(isa.clone(), cranelift_module::default_libcall_names()); + // Register all the helpers + for (k, v) in helpers.iter() { + let name = format!("helper_{}", k); + jit_builder.symbol(name, (*v) as usize as *const u8); + } + + let mut module = JITModule::new(jit_builder); + + let registers = (0..11) + .map(|i| Variable::new(i)) + .collect::>() + .try_into() + .unwrap(); + + Self { + isa, + module, + helpers, + helper_func_refs: HashMap::new(), + insn_blocks: BTreeMap::new(), + insn_targets: BTreeMap::new(), + filled_blocks: HashSet::new(), + registers, + mem_start: Variable::new(11), + mem_end: Variable::new(12), + mbuf_start: Variable::new(13), + mbuf_end: Variable::new(14), + stack_start: Variable::new(15), + stack_end: Variable::new(16), + } + } + + pub(crate) fn compile_function(mut self, prog: &[u8]) -> Result { + let name = "main"; + // This is not a standard eBPF function! We use an informal ABI with just 4 parameters. + // See [JittedFunction] which is the signature of this function. + // + // Since this function only serves as the entrypoint for the JITed program, it doesen't + // really matter. + let sig = Signature { + params: vec![ + AbiParam::new(I64), + AbiParam::new(I64), + AbiParam::new(I64), + AbiParam::new(I64), + ], + returns: vec![AbiParam::new(I64)], + call_conv: self.isa.default_call_conv(), + }; + + let func_id = self + .module + .declare_function(name, Linkage::Local, &sig) + .unwrap(); + + let mut ctx = self.module.make_context(); + ctx.func = Function::with_name_signature(UserFuncName::testcase(name.as_bytes()), sig); + let mut func_ctx = FunctionBuilderContext::new(); + + { + let mut builder: FunctionBuilder = FunctionBuilder::new(&mut ctx.func, &mut func_ctx); + + let entry = builder.create_block(); + builder.append_block_params_for_function_params(entry); + builder.switch_to_block(entry); + + self.build_cfg(&mut builder, prog)?; + self.build_function_prelude(&mut builder, entry)?; + self.translate_program(&mut builder, prog)?; + + builder.seal_all_blocks(); + builder.finalize(); + } + + self.module.define_function(func_id, &mut ctx).unwrap(); + self.module.finalize_definitions().unwrap(); + self.module.clear_context(&mut ctx); + + Ok(CraneliftProgram::new(self.module, func_id)) + } + + fn build_function_prelude( + &mut self, + bcx: &mut FunctionBuilder, + entry: Block, + ) -> Result<(), Error> { + // Register the VM registers as variables + for var in self.registers.iter() { + bcx.declare_var(*var, I64); + } + + // Register the bounds check variables + bcx.declare_var(self.mem_start, I64); + bcx.declare_var(self.mem_end, I64); + bcx.declare_var(self.mbuf_start, I64); + bcx.declare_var(self.mbuf_end, I64); + bcx.declare_var(self.stack_start, I64); + bcx.declare_var(self.stack_end, I64); + + // Register the helpers + for (k, _) in self.helpers.iter() { + let name = format!("helper_{}", k); + let sig = Signature { + params: vec![ + AbiParam::new(I64), + AbiParam::new(I64), + AbiParam::new(I64), + AbiParam::new(I64), + AbiParam::new(I64), + ], + returns: vec![AbiParam::new(I64)], + call_conv: self.isa.default_call_conv(), + }; + let func_id = self + .module + .declare_function(&name, Linkage::Import, &sig) + .unwrap(); + + let func_ref = self.module.declare_func_in_func(func_id, bcx.func); + self.helper_func_refs.insert(*k, func_ref); + } + + // Register the stack + let ss = bcx.create_sized_stack_slot(StackSlotData { + kind: StackSlotKind::ExplicitSlot, + size: STACK_SIZE as u32, + }); + let addr_ty = self.isa.pointer_type(); + let stack_addr = bcx.ins().stack_addr(addr_ty, ss, STACK_SIZE as i32); + bcx.def_var(self.registers[10], stack_addr); + + // Initialize the bounds check variables + let stack_start = bcx.ins().stack_addr(addr_ty, ss, 0); + bcx.def_var(self.stack_start, stack_start); + let stack_end = bcx.ins().stack_addr(addr_ty, ss, STACK_SIZE as i32); + bcx.def_var(self.stack_end, stack_end); + + // This is our internal ABI where the first 2 params are the memory + let mem_start = bcx.block_params(entry)[0]; + let mem_len = bcx.block_params(entry)[1]; + let mem_end = bcx.ins().iadd(mem_start, mem_len); + bcx.def_var(self.mem_start, mem_start); + bcx.def_var(self.mem_end, mem_end); + + // And the next 2 are the mbuf + let mbuf_start = bcx.block_params(entry)[2]; + let mbuf_len = bcx.block_params(entry)[3]; + let mbuf_end = bcx.ins().iadd(mbuf_start, mbuf_len); + bcx.def_var(self.mbuf_start, mbuf_start); + bcx.def_var(self.mbuf_end, mbuf_end); + + // The ABI for eBPF specifies that R1 must contain either the memory, or mbuff pointer + // If the mbuf length is non-zero, then we use that, otherwise we use the memory pointer + let mbuf_exists = bcx.ins().icmp_imm(IntCC::NotEqual, mbuf_len, 0); + let mem_or_mbuf = bcx.ins().select(mbuf_exists, mbuf_start, mem_start); + bcx.def_var(self.registers[1], mem_or_mbuf); + + // R2 should contain the length of the memory or mbuf + // At least ebpf-conformance tests expect this + let mem_or_mbuf_len = bcx.ins().select(mbuf_exists, mbuf_len, mem_len); + bcx.def_var(self.registers[2], mem_or_mbuf_len); + + // Insert the *actual* initial block + let program_entry = bcx.create_block(); + bcx.ins().jump(program_entry, &[]); + self.filled_blocks.insert(bcx.current_block().unwrap()); + self.insn_blocks.insert(0, program_entry); + + Ok(()) + } + + fn translate_program(&mut self, bcx: &mut FunctionBuilder, prog: &[u8]) -> Result<(), Error> { + let mut insn_ptr: usize = 0; + while insn_ptr * ebpf::INSN_SIZE < prog.len() { + let insn = ebpf::get_insn(prog, insn_ptr); + + // If this instruction is on a new block switch to it. + if let Some(block) = self.insn_blocks.get(&(insn_ptr as u32)) { + // Blocks must have a terminator instruction at the end before we switch away from them + let current_block = bcx.current_block().unwrap(); + if !self.filled_blocks.contains(¤t_block) { + bcx.ins().jump(*block, &[]); + } + + bcx.switch_to_block(*block); + } + + // Set the source location for the instruction + bcx.set_srcloc(SourceLoc::new(insn_ptr as u32)); + + match insn.opc { + // BPF_LD class + // LD_ABS_* and LD_IND_* are supposed to load pointer to data from metadata buffer. + // Since this pointer is constant, and since we already know it (mem), do not + // bother re-fetching it, just use mem already. + ebpf::LD_ABS_B + | ebpf::LD_ABS_H + | ebpf::LD_ABS_W + | ebpf::LD_ABS_DW + | ebpf::LD_IND_B + | ebpf::LD_IND_H + | ebpf::LD_IND_W + | ebpf::LD_IND_DW => { + let ty = match insn.opc { + ebpf::LD_ABS_B | ebpf::LD_IND_B => I8, + ebpf::LD_ABS_H | ebpf::LD_IND_H => I16, + ebpf::LD_ABS_W | ebpf::LD_IND_W => I32, + ebpf::LD_ABS_DW | ebpf::LD_IND_DW => I64, + _ => unreachable!(), + }; + + // Both instructions add the imm part of the instruction to the pointer + let ptr = bcx.use_var(self.mem_start); + let offset = bcx + .ins() + .iconst(self.isa.pointer_type(), insn.imm as u32 as i64); + let addr = bcx.ins().iadd(ptr, offset); + + // IND instructions additionally add the value of the source register + let is_ind = (insn.opc & BPF_IND) != 0; + let addr = if is_ind { + let src_reg = self.insn_src(bcx, &insn); + bcx.ins().iadd(addr, src_reg) + } else { + addr + }; + + // The offset here has already been added to the pointer, so we pass 0 + let loaded = self.reg_load(bcx, ty, addr, 0); + + let ext = if ty != I64 { + bcx.ins().uextend(I64, loaded) + } else { + loaded + }; + + self.set_dst(bcx, &insn, ext); + } + ebpf::LD_DW_IMM => { + insn_ptr += 1; + let next_insn = ebpf::get_insn(prog, insn_ptr); + + let imm = (((insn.imm as u32) as u64) + ((next_insn.imm as u64) << 32)) as i64; + let iconst = bcx.ins().iconst(I64, imm); + self.set_dst(bcx, &insn, iconst); + } + + // BPF_LDX class + ebpf::LD_B_REG | ebpf::LD_H_REG | ebpf::LD_W_REG | ebpf::LD_DW_REG => { + let ty = match insn.opc { + ebpf::LD_B_REG => I8, + ebpf::LD_H_REG => I16, + ebpf::LD_W_REG => I32, + ebpf::LD_DW_REG => I64, + _ => unreachable!(), + }; + + let base = self.insn_src(bcx, &insn); + let loaded = self.reg_load(bcx, ty, base, insn.off); + + let ext = if ty != I64 { + bcx.ins().uextend(I64, loaded) + } else { + loaded + }; + + self.set_dst(bcx, &insn, ext); + } + + // BPF_ST and BPF_STX class + ebpf::ST_B_IMM + | ebpf::ST_H_IMM + | ebpf::ST_W_IMM + | ebpf::ST_DW_IMM + | ebpf::ST_B_REG + | ebpf::ST_H_REG + | ebpf::ST_W_REG + | ebpf::ST_DW_REG => { + let ty = match insn.opc { + ebpf::ST_B_IMM | ebpf::ST_B_REG => I8, + ebpf::ST_H_IMM | ebpf::ST_H_REG => I16, + ebpf::ST_W_IMM | ebpf::ST_W_REG => I32, + ebpf::ST_DW_IMM | ebpf::ST_DW_REG => I64, + _ => unreachable!(), + }; + let is_imm = match insn.opc { + ebpf::ST_B_IMM | ebpf::ST_H_IMM | ebpf::ST_W_IMM | ebpf::ST_DW_IMM => true, + ebpf::ST_B_REG | ebpf::ST_H_REG | ebpf::ST_W_REG | ebpf::ST_DW_REG => false, + _ => unreachable!(), + }; + + let value = if is_imm { + self.insn_imm64(bcx, &insn) + } else { + self.insn_src(bcx, &insn) + }; + + let narrow = if ty != I64 { + bcx.ins().ireduce(ty, value) + } else { + value + }; + + let base = self.insn_dst(bcx, &insn); + self.reg_store(bcx, ty, base, insn.off, narrow); + } + + ebpf::ST_W_XADD => unimplemented!(), + ebpf::ST_DW_XADD => unimplemented!(), + + // BPF_ALU class + // TODO Check how overflow works in kernel. Should we &= U32MAX all src register value + // before we do the operation? + // Cf ((0x11 << 32) - (0x1 << 32)) as u32 VS ((0x11 << 32) as u32 - (0x1 << 32) as u32 + ebpf::ADD32_IMM => { + let src = self.insn_dst32(bcx, &insn); + let imm = self.insn_imm32(bcx, &insn); + let res = bcx.ins().iadd(src, imm); + self.set_dst32(bcx, &insn, res); + } + ebpf::ADD32_REG => { + //((reg[_dst] & U32MAX) + (reg[_src] & U32MAX)) & U32MAX, + let lhs = self.insn_dst32(bcx, &insn); + let rhs = self.insn_src32(bcx, &insn); + let res = bcx.ins().iadd(lhs, rhs); + self.set_dst32(bcx, &insn, res); + } + ebpf::SUB32_IMM => { + // reg[_dst] = (reg[_dst] as i32).wrapping_sub(insn.imm) as u64, + let src = self.insn_dst32(bcx, &insn); + let imm = self.insn_imm32(bcx, &insn); + let res = bcx.ins().isub(src, imm); + self.set_dst32(bcx, &insn, res); + } + ebpf::SUB32_REG => { + // reg[_dst] = (reg[_dst] as i32).wrapping_sub(reg[_src] as i32) as u64, + let lhs = self.insn_dst32(bcx, &insn); + let rhs = self.insn_src32(bcx, &insn); + let res = bcx.ins().isub(lhs, rhs); + self.set_dst32(bcx, &insn, res); + } + ebpf::MUL32_IMM => { + // reg[_dst] = (reg[_dst] as i32).wrapping_mul(insn.imm) as u64, + let src = self.insn_dst32(bcx, &insn); + let imm = self.insn_imm32(bcx, &insn); + let res = bcx.ins().imul(src, imm); + self.set_dst32(bcx, &insn, res); + } + ebpf::MUL32_REG => { + // reg[_dst] = (reg[_dst] as i32).wrapping_mul(reg[_src] as i32) as u64, + let lhs = self.insn_dst32(bcx, &insn); + let rhs = self.insn_src32(bcx, &insn); + let res = bcx.ins().imul(lhs, rhs); + self.set_dst32(bcx, &insn, res); + } + ebpf::DIV32_IMM => { + // reg[_dst] = (reg[_dst] as u32 / insn.imm as u32) as u64, + let res = if insn.imm == 0 { + bcx.ins().iconst(I32, 0) + } else { + let imm = self.insn_imm32(bcx, &insn); + let src = self.insn_dst32(bcx, &insn); + bcx.ins().udiv(src, imm) + }; + self.set_dst32(bcx, &insn, res); + } + ebpf::DIV32_REG => { + // reg[_dst] = (reg[_dst] as u32 / reg[_src] as u32) as u64, + let zero = bcx.ins().iconst(I32, 0); + let one = bcx.ins().iconst(I32, 1); + + let lhs = self.insn_dst32(bcx, &insn); + let rhs = self.insn_src32(bcx, &insn); + + let rhs_is_zero = bcx.ins().icmp(IntCC::Equal, rhs, zero); + let safe_rhs = bcx.ins().select(rhs_is_zero, one, rhs); + let div_res = bcx.ins().udiv(lhs, safe_rhs); + + let res = bcx.ins().select(rhs_is_zero, zero, div_res); + self.set_dst32(bcx, &insn, res); + } + ebpf::OR32_IMM => { + // reg[_dst] = (reg[_dst] as u32 | insn.imm as u32) as u64, + let src = self.insn_dst32(bcx, &insn); + let imm = self.insn_imm32(bcx, &insn); + let res = bcx.ins().bor(src, imm); + self.set_dst32(bcx, &insn, res); + } + ebpf::OR32_REG => { + // reg[_dst] = (reg[_dst] as u32 | reg[_src] as u32) as u64, + let lhs = self.insn_dst32(bcx, &insn); + let rhs = self.insn_src32(bcx, &insn); + let res = bcx.ins().bor(lhs, rhs); + self.set_dst32(bcx, &insn, res); + } + ebpf::AND32_IMM => { + // reg[_dst] = (reg[_dst] as u32 & insn.imm as u32) as u64, + let src = self.insn_dst32(bcx, &insn); + let imm = self.insn_imm32(bcx, &insn); + let res = bcx.ins().band(src, imm); + self.set_dst32(bcx, &insn, res); + } + ebpf::AND32_REG => { + // reg[_dst] = (reg[_dst] as u32 & reg[_src] as u32) as u64, + let lhs = self.insn_dst32(bcx, &insn); + let rhs = self.insn_src32(bcx, &insn); + let res = bcx.ins().band(lhs, rhs); + self.set_dst32(bcx, &insn, res); + } + ebpf::LSH32_IMM => { + // reg[_dst] = (reg[_dst] as u32).wrapping_shl(insn.imm as u32) as u64, + let src = self.insn_dst32(bcx, &insn); + let imm = self.insn_imm32(bcx, &insn); + let res = bcx.ins().ishl(src, imm); + self.set_dst32(bcx, &insn, res); + } + ebpf::LSH32_REG => { + // reg[_dst] = (reg[_dst] as u32).wrapping_shl(reg[_src] as u32) as u64, + let lhs = self.insn_dst32(bcx, &insn); + let rhs = self.insn_src32(bcx, &insn); + let res = bcx.ins().ishl(lhs, rhs); + self.set_dst32(bcx, &insn, res); + } + ebpf::RSH32_IMM => { + // reg[_dst] = (reg[_dst] as u32).wrapping_shr(insn.imm as u32) as u64, + let src = self.insn_dst32(bcx, &insn); + let imm = self.insn_imm32(bcx, &insn); + let res = bcx.ins().ushr(src, imm); + self.set_dst32(bcx, &insn, res); + } + ebpf::RSH32_REG => { + // reg[_dst] = (reg[_dst] as u32).wrapping_shr(reg[_src] as u32) as u64, + let lhs = self.insn_dst32(bcx, &insn); + let rhs = self.insn_src32(bcx, &insn); + let res = bcx.ins().ushr(lhs, rhs); + self.set_dst32(bcx, &insn, res); + } + ebpf::NEG32 => { + // { reg[_dst] = (reg[_dst] as i32).wrapping_neg() as u64; reg[_dst] &= U32MAX; }, + let src = self.insn_dst32(bcx, &insn); + let res = bcx.ins().ineg(src); + // TODO: Do we need to mask the result? + self.set_dst32(bcx, &insn, res); + } + ebpf::MOD32_IMM => { + // reg[_dst] = (reg[_dst] as u32 % insn.imm as u32) as u64, + + if insn.imm != 0 { + let imm = self.insn_imm32(bcx, &insn); + let src = self.insn_dst32(bcx, &insn); + let res = bcx.ins().urem(src, imm); + self.set_dst32(bcx, &insn, res); + } + } + ebpf::MOD32_REG => { + // reg[_dst] = (reg[_dst] as u32 % reg[_src] as u32) as u64, + let zero = bcx.ins().iconst(I32, 0); + let one = bcx.ins().iconst(I32, 1); + + let lhs = self.insn_dst32(bcx, &insn); + let rhs = self.insn_src32(bcx, &insn); + + let rhs_is_zero = bcx.ins().icmp(IntCC::Equal, rhs, zero); + let safe_rhs = bcx.ins().select(rhs_is_zero, one, rhs); + let div_res = bcx.ins().urem(lhs, safe_rhs); + + let res = bcx.ins().select(rhs_is_zero, lhs, div_res); + self.set_dst32(bcx, &insn, res); + } + ebpf::XOR32_IMM => { + // reg[_dst] = (reg[_dst] as u32 ^ insn.imm as u32) as u64, + let src = self.insn_dst32(bcx, &insn); + let imm = self.insn_imm32(bcx, &insn); + let res = bcx.ins().bxor(src, imm); + self.set_dst32(bcx, &insn, res); + } + ebpf::XOR32_REG => { + // reg[_dst] = (reg[_dst] as u32 ^ reg[_src] as u32) as u64, + let lhs = self.insn_dst32(bcx, &insn); + let rhs = self.insn_src32(bcx, &insn); + let res = bcx.ins().bxor(lhs, rhs); + self.set_dst32(bcx, &insn, res); + } + ebpf::MOV32_IMM => { + let imm = self.insn_imm32(bcx, &insn); + self.set_dst32(bcx, &insn, imm); + } + ebpf::MOV32_REG => { + // reg[_dst] = (reg[_src] as u32) as u64, + let src = self.insn_src32(bcx, &insn); + self.set_dst32(bcx, &insn, src); + } + ebpf::ARSH32_IMM => { + // { reg[_dst] = (reg[_dst] as i32).wrapping_shr(insn.imm as u32) as u64; reg[_dst] &= U32MAX; }, + let src = self.insn_dst32(bcx, &insn); + let imm = self.insn_imm32(bcx, &insn); + let res = bcx.ins().sshr(src, imm); + self.set_dst32(bcx, &insn, res); + } + ebpf::ARSH32_REG => { + // { reg[_dst] = (reg[_dst] as i32).wrapping_shr(reg[_src] as u32) as u64; reg[_dst] &= U32MAX; }, + let lhs = self.insn_dst32(bcx, &insn); + let rhs = self.insn_src32(bcx, &insn); + let res = bcx.ins().sshr(lhs, rhs); + self.set_dst32(bcx, &insn, res); + } + + ebpf::BE | ebpf::LE => { + let should_swap = match insn.opc { + ebpf::BE => self.isa.endianness() == Endianness::Little, + ebpf::LE => self.isa.endianness() == Endianness::Big, + _ => unreachable!(), + }; + + let ty: Type = match insn.imm { + 16 => I16, + 32 => I32, + 64 => I64, + _ => unreachable!(), + }; + + if should_swap { + let src = self.insn_dst(bcx, &insn); + let src_narrow = if ty != I64 { + bcx.ins().ireduce(ty, src) + } else { + src + }; + + let res = bcx.ins().bswap(src_narrow); + let res_wide = if ty != I64 { + bcx.ins().uextend(I64, res) + } else { + res + }; + + self.set_dst(bcx, &insn, res_wide); + } + } + + // BPF_ALU64 class + ebpf::ADD64_IMM => { + // reg[_dst] = reg[_dst].wrapping_add(insn.imm as u64), + let imm = self.insn_imm64(bcx, &insn); + let src = self.insn_dst(bcx, &insn); + let res = bcx.ins().iadd(src, imm); + self.set_dst(bcx, &insn, res); + } + ebpf::ADD64_REG => { + // reg[_dst] = reg[_dst].wrapping_add(reg[_src]), + let lhs = self.insn_dst(bcx, &insn); + let rhs = self.insn_src(bcx, &insn); + let res = bcx.ins().iadd(lhs, rhs); + self.set_dst(bcx, &insn, res); + } + ebpf::SUB64_IMM => { + // reg[_dst] = reg[_dst].wrapping_sub(insn.imm as u64), + let imm = self.insn_imm64(bcx, &insn); + let src = self.insn_dst(bcx, &insn); + let res = bcx.ins().isub(src, imm); + self.set_dst(bcx, &insn, res); + } + ebpf::SUB64_REG => { + // reg[_dst] = reg[_dst].wrapping_sub(reg[_src]), + let lhs = self.insn_dst(bcx, &insn); + let rhs = self.insn_src(bcx, &insn); + let res = bcx.ins().isub(lhs, rhs); + self.set_dst(bcx, &insn, res); + } + ebpf::MUL64_IMM => { + // reg[_dst] = reg[_dst].wrapping_mul(insn.imm as u64), + let imm = self.insn_imm64(bcx, &insn); + let src = self.insn_dst(bcx, &insn); + let res = bcx.ins().imul(src, imm); + self.set_dst(bcx, &insn, res); + } + ebpf::MUL64_REG => { + // reg[_dst] = reg[_dst].wrapping_mul(reg[_src]), + let lhs = self.insn_dst(bcx, &insn); + let rhs = self.insn_src(bcx, &insn); + let res = bcx.ins().imul(lhs, rhs); + self.set_dst(bcx, &insn, res); + } + ebpf::DIV64_IMM => { + // reg[_dst] /= insn.imm as u64, + let res = if insn.imm == 0 { + bcx.ins().iconst(I64, 0) + } else { + let imm = self.insn_imm64(bcx, &insn); + let src = self.insn_dst(bcx, &insn); + bcx.ins().udiv(src, imm) + }; + self.set_dst(bcx, &insn, res); + } + ebpf::DIV64_REG => { + // reg[_dst] /= reg[_src], if reg[_src] != 0 + // reg[_dst] = 0, if reg[_src] == 0 + let zero = bcx.ins().iconst(I64, 0); + let one = bcx.ins().iconst(I64, 1); + + let lhs = self.insn_dst(bcx, &insn); + let rhs = self.insn_src(bcx, &insn); + + let rhs_is_zero = bcx.ins().icmp(IntCC::Equal, rhs, zero); + let safe_rhs = bcx.ins().select(rhs_is_zero, one, rhs); + let div_res = bcx.ins().udiv(lhs, safe_rhs); + + let res = bcx.ins().select(rhs_is_zero, zero, div_res); + self.set_dst(bcx, &insn, res); + } + ebpf::MOD64_IMM => { + // reg[_dst] %= insn.imm as u64, + + if insn.imm != 0 { + let imm = self.insn_imm64(bcx, &insn); + let src = self.insn_dst(bcx, &insn); + let res = bcx.ins().urem(src, imm); + self.set_dst(bcx, &insn, res); + }; + } + ebpf::MOD64_REG => { + // reg[_dst] %= reg[_src], if reg[_src] != 0 + + let zero = bcx.ins().iconst(I64, 0); + let one = bcx.ins().iconst(I64, 1); + + let lhs = self.insn_dst(bcx, &insn); + let rhs = self.insn_src(bcx, &insn); + + let rhs_is_zero = bcx.ins().icmp(IntCC::Equal, rhs, zero); + let safe_rhs = bcx.ins().select(rhs_is_zero, one, rhs); + let div_res = bcx.ins().urem(lhs, safe_rhs); + + let res = bcx.ins().select(rhs_is_zero, lhs, div_res); + self.set_dst(bcx, &insn, res); + } + ebpf::OR64_IMM => { + // reg[_dst] |= insn.imm as u64, + let imm = self.insn_imm64(bcx, &insn); + let src = self.insn_dst(bcx, &insn); + let res = bcx.ins().bor(src, imm); + self.set_dst(bcx, &insn, res); + } + ebpf::OR64_REG => { + // reg[_dst] |= reg[_src], + let lhs = self.insn_dst(bcx, &insn); + let rhs = self.insn_src(bcx, &insn); + let res = bcx.ins().bor(lhs, rhs); + self.set_dst(bcx, &insn, res); + } + ebpf::AND64_IMM => { + // reg[_dst] &= insn.imm as u64, + let imm = self.insn_imm64(bcx, &insn); + let src = self.insn_dst(bcx, &insn); + let res = bcx.ins().band(src, imm); + self.set_dst(bcx, &insn, res); + } + ebpf::AND64_REG => { + // reg[_dst] &= reg[_src], + let lhs = self.insn_dst(bcx, &insn); + let rhs = self.insn_src(bcx, &insn); + let res = bcx.ins().band(lhs, rhs); + self.set_dst(bcx, &insn, res); + } + ebpf::LSH64_IMM => { + // reg[_dst] <<= insn.imm as u64, + let imm = self.insn_imm64(bcx, &insn); + let src = self.insn_dst(bcx, &insn); + let res = bcx.ins().ishl(src, imm); + self.set_dst(bcx, &insn, res); + } + ebpf::LSH64_REG => { + // reg[_dst] <<= reg[_src], + let lhs = self.insn_dst(bcx, &insn); + let rhs = self.insn_src(bcx, &insn); + let res = bcx.ins().ishl(lhs, rhs); + self.set_dst(bcx, &insn, res); + } + ebpf::RSH64_IMM => { + // reg[_dst] >>= insn.imm as u64, + let imm = self.insn_imm64(bcx, &insn); + let src = self.insn_dst(bcx, &insn); + let res = bcx.ins().ushr(src, imm); + self.set_dst(bcx, &insn, res); + } + ebpf::RSH64_REG => { + // reg[_dst] >>= reg[_src], + let lhs = self.insn_dst(bcx, &insn); + let rhs = self.insn_src(bcx, &insn); + let res = bcx.ins().ushr(lhs, rhs); + self.set_dst(bcx, &insn, res); + } + ebpf::NEG64 => { + // reg[_dst] = -(reg[_dst] as i64) as u64, + let src = self.insn_dst(bcx, &insn); + let res = bcx.ins().ineg(src); + self.set_dst(bcx, &insn, res); + } + ebpf::XOR64_IMM => { + // reg[_dst] ^= insn.imm as u64, + let imm = self.insn_imm64(bcx, &insn); + let src = self.insn_dst(bcx, &insn); + let res = bcx.ins().bxor(src, imm); + self.set_dst(bcx, &insn, res); + } + ebpf::XOR64_REG => { + // reg[_dst] ^= reg[_src], + let lhs = self.insn_dst(bcx, &insn); + let rhs = self.insn_src(bcx, &insn); + let res = bcx.ins().bxor(lhs, rhs); + self.set_dst(bcx, &insn, res); + } + ebpf::MOV64_IMM => { + // reg[_dst] = insn.imm as u64, + let imm = self.insn_imm64(bcx, &insn); + bcx.def_var(self.registers[insn.dst as usize], imm); + } + ebpf::MOV64_REG => { + // reg[_dst] = reg[_src], + let src = self.insn_src(bcx, &insn); + bcx.def_var(self.registers[insn.dst as usize], src); + } + ebpf::ARSH64_IMM => { + // reg[_dst] = (reg[_dst] as i64 >> insn.imm) as u64, + let imm = self.insn_imm64(bcx, &insn); + let src = self.insn_dst(bcx, &insn); + let res = bcx.ins().sshr(src, imm); + self.set_dst(bcx, &insn, res); + } + ebpf::ARSH64_REG => { + // reg[_dst] = (reg[_dst] as i64 >> reg[_src]) as u64, + let lhs = self.insn_dst(bcx, &insn); + let rhs = self.insn_src(bcx, &insn); + let res = bcx.ins().sshr(lhs, rhs); + self.set_dst(bcx, &insn, res); + } + + // BPF_JMP & BPF_JMP32 class + ebpf::JA => { + let (_, target_block) = self.insn_targets[&(insn_ptr as u32)]; + + bcx.ins().jump(target_block, &[]); + self.filled_blocks.insert(bcx.current_block().unwrap()); + } + ebpf::JEQ_IMM + | ebpf::JEQ_REG + | ebpf::JGT_IMM + | ebpf::JGT_REG + | ebpf::JGE_IMM + | ebpf::JGE_REG + | ebpf::JLT_IMM + | ebpf::JLT_REG + | ebpf::JLE_IMM + | ebpf::JLE_REG + | ebpf::JNE_IMM + | ebpf::JNE_REG + | ebpf::JSGT_IMM + | ebpf::JSGT_REG + | ebpf::JSGE_IMM + | ebpf::JSGE_REG + | ebpf::JSLT_IMM + | ebpf::JSLT_REG + | ebpf::JSLE_IMM + | ebpf::JSLE_REG + | ebpf::JSET_IMM + | ebpf::JSET_REG + | ebpf::JEQ_IMM32 + | ebpf::JEQ_REG32 + | ebpf::JGT_IMM32 + | ebpf::JGT_REG32 + | ebpf::JGE_IMM32 + | ebpf::JGE_REG32 + | ebpf::JLT_IMM32 + | ebpf::JLT_REG32 + | ebpf::JLE_IMM32 + | ebpf::JLE_REG32 + | ebpf::JNE_IMM32 + | ebpf::JNE_REG32 + | ebpf::JSGT_IMM32 + | ebpf::JSGT_REG32 + | ebpf::JSGE_IMM32 + | ebpf::JSGE_REG32 + | ebpf::JSLT_IMM32 + | ebpf::JSLT_REG32 + | ebpf::JSLE_IMM32 + | ebpf::JSLE_REG32 + | ebpf::JSET_IMM32 + | ebpf::JSET_REG32 => { + let (fallthrough, target) = self.insn_targets[&(insn_ptr as u32)]; + + let is_reg = (insn.opc & BPF_X) != 0; + let is_32 = (insn.opc & BPF_JMP32) != 0; + let intcc = match insn.opc { + c if (c & BPF_ALU_OP_MASK) == BPF_JEQ => IntCC::Equal, + c if (c & BPF_ALU_OP_MASK) == BPF_JNE => IntCC::NotEqual, + c if (c & BPF_ALU_OP_MASK) == BPF_JGT => IntCC::UnsignedGreaterThan, + c if (c & BPF_ALU_OP_MASK) == BPF_JGE => IntCC::UnsignedGreaterThanOrEqual, + c if (c & BPF_ALU_OP_MASK) == BPF_JLT => IntCC::UnsignedLessThan, + c if (c & BPF_ALU_OP_MASK) == BPF_JLE => IntCC::UnsignedLessThanOrEqual, + c if (c & BPF_ALU_OP_MASK) == BPF_JSGT => IntCC::SignedGreaterThan, + c if (c & BPF_ALU_OP_MASK) == BPF_JSGE => IntCC::SignedGreaterThanOrEqual, + c if (c & BPF_ALU_OP_MASK) == BPF_JSLT => IntCC::SignedLessThan, + c if (c & BPF_ALU_OP_MASK) == BPF_JSLE => IntCC::SignedLessThanOrEqual, + // JSET is handled specially below + c if (c & BPF_ALU_OP_MASK) == BPF_JSET => IntCC::NotEqual, + _ => unreachable!(), + }; + + let lhs = if is_32 { + self.insn_dst32(bcx, &insn) + } else { + self.insn_dst(bcx, &insn) + }; + let rhs = match (is_reg, is_32) { + (true, false) => self.insn_src(bcx, &insn), + (true, true) => self.insn_src32(bcx, &insn), + (false, false) => self.insn_imm64(bcx, &insn), + (false, true) => self.insn_imm32(bcx, &insn), + }; + + let cmp_res = if (insn.opc & BPF_ALU_OP_MASK) == BPF_JSET { + bcx.ins().band(lhs, rhs) + } else { + bcx.ins().icmp(intcc, lhs, rhs) + }; + bcx.ins().brif(cmp_res, target, &[], fallthrough, &[]); + self.filled_blocks.insert(bcx.current_block().unwrap()); + } + + // Do not delegate the check to the verifier, since registered functions can be + // changed after the program has been verified. + ebpf::CALL => { + let func_ref = self + .helper_func_refs + .get(&(insn.imm as u32)) + .copied() + .ok_or_else(|| { + Error::new( + ErrorKind::Other, + format!( + "[CRANELIFT] Error: unknown helper function (id: {:#x})", + insn.imm as u32 + ), + ) + })?; + + let arg0 = bcx.use_var(self.registers[1]); + let arg1 = bcx.use_var(self.registers[2]); + let arg2 = bcx.use_var(self.registers[3]); + let arg3 = bcx.use_var(self.registers[4]); + let arg4 = bcx.use_var(self.registers[5]); + + let call = bcx.ins().call(func_ref, &[arg0, arg1, arg2, arg3, arg4]); + let ret = bcx.inst_results(call)[0]; + self.set_dst(bcx, &insn, ret); + } + ebpf::TAIL_CALL => unimplemented!(), + ebpf::EXIT => { + let ret = bcx.use_var(self.registers[0]); + bcx.ins().return_(&[ret]); + self.filled_blocks.insert(bcx.current_block().unwrap()); + } + _ => unimplemented!("inst: {:?}", insn), + } + + insn_ptr += 1; + } + + Ok(()) + } + + fn insn_imm64(&mut self, bcx: &mut FunctionBuilder, insn: &Insn) -> Value { + bcx.ins().iconst(I64, insn.imm as u64 as i64) + } + fn insn_imm32(&mut self, bcx: &mut FunctionBuilder, insn: &Insn) -> Value { + bcx.ins().iconst(I32, insn.imm as u32 as u64 as i64) + } + + fn insn_dst(&mut self, bcx: &mut FunctionBuilder, insn: &Insn) -> Value { + bcx.use_var(self.registers[insn.dst as usize]) + } + fn insn_dst32(&mut self, bcx: &mut FunctionBuilder, insn: &Insn) -> Value { + let dst = self.insn_dst(bcx, insn); + bcx.ins().ireduce(I32, dst) + } + + fn insn_src(&mut self, bcx: &mut FunctionBuilder, insn: &Insn) -> Value { + bcx.use_var(self.registers[insn.src as usize]) + } + fn insn_src32(&mut self, bcx: &mut FunctionBuilder, insn: &Insn) -> Value { + let src = self.insn_src(bcx, insn); + bcx.ins().ireduce(I32, src) + } + + fn set_dst(&mut self, bcx: &mut FunctionBuilder, insn: &Insn, val: Value) { + bcx.def_var(self.registers[insn.dst as usize], val); + } + fn set_dst32(&mut self, bcx: &mut FunctionBuilder, insn: &Insn, val: Value) { + let val32 = bcx.ins().uextend(I64, val); + self.set_dst(bcx, insn, val32); + } + + fn reg_load(&mut self, bcx: &mut FunctionBuilder, ty: Type, base: Value, offset: i16) -> Value { + self.insert_bounds_check(bcx, ty, base, offset); + + let mut flags = MemFlags::new(); + flags.set_endianness(Endianness::Little); + + bcx.ins().load(ty, flags, base, offset as i32) + } + fn reg_store( + &mut self, + bcx: &mut FunctionBuilder, + ty: Type, + base: Value, + offset: i16, + val: Value, + ) { + self.insert_bounds_check(bcx, ty, base, offset); + + let mut flags = MemFlags::new(); + flags.set_endianness(Endianness::Little); + + bcx.ins().store(flags, val, base, offset as i32); + } + + /// Inserts a bounds check for a memory access + /// + /// This emits a conditional trap if the access is out of bounds for any of the known + /// valid memory regions. These are the stack, the memory, and the mbuf. + fn insert_bounds_check( + &mut self, + bcx: &mut FunctionBuilder, + ty: Type, + base: Value, + offset: i16, + ) { + let access_size = bcx.ins().iconst(I64, ty.bytes() as i64); + + let offset = bcx.ins().iconst(I64, offset as i64); + let start_addr = bcx.ins().iadd(base, offset); + let end_addr = bcx.ins().iadd(start_addr, access_size); + + let does_not_overflow = + bcx.ins() + .icmp(IntCC::UnsignedGreaterThanOrEqual, end_addr, start_addr); + + // Check if it's a valid stack access + let stack_start = bcx.use_var(self.stack_start); + let stack_end = bcx.use_var(self.stack_end); + let stack_start_valid = + bcx.ins() + .icmp(IntCC::UnsignedGreaterThanOrEqual, start_addr, stack_start); + let stack_end_valid = bcx + .ins() + .icmp(IntCC::UnsignedLessThanOrEqual, end_addr, stack_end); + let stack_valid = bcx.ins().band(stack_start_valid, stack_end_valid); + + // Check if it's a valid memory access + let mem_start = bcx.use_var(self.mem_start); + let mem_end = bcx.use_var(self.mem_end); + let has_mem = bcx.ins().icmp_imm(IntCC::NotEqual, mem_start, 0); + let mem_start_valid = + bcx.ins() + .icmp(IntCC::UnsignedGreaterThanOrEqual, start_addr, mem_start); + let mem_end_valid = bcx + .ins() + .icmp(IntCC::UnsignedLessThanOrEqual, end_addr, mem_end); + + let mem_valid = bcx.ins().band(mem_start_valid, mem_end_valid); + let mem_valid = bcx.ins().band(mem_valid, has_mem); + + // Check if it's a valid mbuf access + let mbuf_start = bcx.use_var(self.mbuf_start); + let mbuf_end = bcx.use_var(self.mbuf_end); + let has_mbuf = bcx.ins().icmp_imm(IntCC::NotEqual, mbuf_start, 0); + let mbuf_start_valid = + bcx.ins() + .icmp(IntCC::UnsignedGreaterThanOrEqual, start_addr, mbuf_start); + let mbuf_end_valid = bcx + .ins() + .icmp(IntCC::UnsignedLessThanOrEqual, end_addr, mbuf_end); + let mbuf_valid = bcx.ins().band(mbuf_start_valid, mbuf_end_valid); + let mbuf_valid = bcx.ins().band(mbuf_valid, has_mbuf); + + // Join all of these checks together and trap if any of them fails + + // We need it to be valid to at least one region of memory + let valid_region = bcx.ins().bor(stack_valid, mem_valid); + let valid_region = bcx.ins().bor(valid_region, mbuf_valid); + + // And that it does not overflow + let valid = bcx.ins().band(does_not_overflow, valid_region); + + // TODO: We can potentially throw a custom trap code here to indicate + // which check failed. + bcx.ins().trapz(valid, TrapCode::HeapOutOfBounds); + } + + /// Analyze the program and build the CFG + /// + /// We do this because cranelift does not allow us to switch back to a previously + /// filled block and add instructions to it. So we can't split the program as we + /// translate it. + fn build_cfg(&mut self, bcx: &mut FunctionBuilder, prog: &[u8]) -> Result<(), Error> { + let mut insn_ptr: usize = 0; + while insn_ptr * ebpf::INSN_SIZE < prog.len() { + let insn = ebpf::get_insn(prog, insn_ptr); + + match insn.opc { + // This instruction consumes two opcodes + ebpf::LD_DW_IMM => { + insn_ptr += 1; + } + + ebpf::JA + | ebpf::JEQ_IMM + | ebpf::JEQ_REG + | ebpf::JGT_IMM + | ebpf::JGT_REG + | ebpf::JGE_IMM + | ebpf::JGE_REG + | ebpf::JLT_IMM + | ebpf::JLT_REG + | ebpf::JLE_IMM + | ebpf::JLE_REG + | ebpf::JNE_IMM + | ebpf::JNE_REG + | ebpf::JSGT_IMM + | ebpf::JSGT_REG + | ebpf::JSGE_IMM + | ebpf::JSGE_REG + | ebpf::JSLT_IMM + | ebpf::JSLT_REG + | ebpf::JSLE_IMM + | ebpf::JSLE_REG + | ebpf::JSET_IMM + | ebpf::JSET_REG + | ebpf::JEQ_IMM32 + | ebpf::JEQ_REG32 + | ebpf::JGT_IMM32 + | ebpf::JGT_REG32 + | ebpf::JGE_IMM32 + | ebpf::JGE_REG32 + | ebpf::JLT_IMM32 + | ebpf::JLT_REG32 + | ebpf::JLE_IMM32 + | ebpf::JLE_REG32 + | ebpf::JNE_IMM32 + | ebpf::JNE_REG32 + | ebpf::JSGT_IMM32 + | ebpf::JSGT_REG32 + | ebpf::JSGE_IMM32 + | ebpf::JSGE_REG32 + | ebpf::JSLT_IMM32 + | ebpf::JSLT_REG32 + | ebpf::JSLE_IMM32 + | ebpf::JSLE_REG32 + | ebpf::JSET_IMM32 + | ebpf::JSET_REG32 + | ebpf::EXIT + | ebpf::TAIL_CALL => { + self.prepare_jump_blocks(bcx, insn_ptr, &insn); + } + _ => {} + } + + insn_ptr += 1; + } + + Ok(()) + } + + fn prepare_jump_blocks(&mut self, bcx: &mut FunctionBuilder, insn_ptr: usize, insn: &Insn) { + let insn_ptr = insn_ptr as u32; + let next_pc: u32 = insn_ptr + 1; + let target_pc: u32 = (insn_ptr as isize + insn.off as isize + 1) + .try_into() + .unwrap(); + + // This is the fallthrough block + let fallthrough_block = *self + .insn_blocks + .entry(next_pc) + .or_insert_with(|| bcx.create_block()); + + // Jump Target + let target_block = *self + .insn_blocks + .entry(target_pc) + .or_insert_with(|| bcx.create_block()); + + // Mark the blocks for this instruction + self.insn_targets + .insert(insn_ptr, (fallthrough_block, target_block)); + } +} + +/// Contains the backing memory for a previously compiled function. +/// +/// Currently this will allways just contain code for a single function, but +/// in the future we might want to support multiple functions per module. +/// +/// Ensures that the backing memory is freed when dropped. +pub struct CraneliftProgram { + module: ManuallyDrop, + + main_id: FuncId, +} + +impl CraneliftProgram { + pub(crate) fn new(module: JITModule, main_id: FuncId) -> Self { + Self { + module: ManuallyDrop::new(module), + main_id, + } + } + + /// We shouldn't allow this function pointer to be exposed outside of this + /// module, since it's not guaranteed to be valid after the module is dropped. + pub(crate) fn get_main_function(&self) -> JittedFunction { + let function_ptr = self.module.get_finalized_function(self.main_id); + unsafe { mem::transmute(function_ptr) } + } + + /// Execute this module by calling the main function + pub fn execute( + &self, + mem_ptr: *mut u8, + mem_len: usize, + mbuff_ptr: *mut u8, + mbuff_len: usize, + ) -> u64 { + let main = self.get_main_function(); + + main(mem_ptr, mem_len, mbuff_ptr, mbuff_len) + } +} + +impl Drop for CraneliftProgram { + fn drop(&mut self) { + // We need to have an owned version of `JITModule` to be able to free + // it's memory. Use `ManuallyDrop` to get the owned `JITModule`. + // + // We can no longer use `module` after this, but since we are `Drop` + // it should be safe. + unsafe { + let module = ManuallyDrop::take(&mut self.module); + module.free_memory() + }; + } +} diff --git a/kernel/crates/rbpf/src/disassembler.rs b/kernel/crates/rbpf/src/disassembler.rs new file mode 100644 index 000000000..6d6572c5c --- /dev/null +++ b/kernel/crates/rbpf/src/disassembler.rs @@ -0,0 +1,807 @@ +// SPDX-License-Identifier: (Apache-2.0 OR MIT) +// Copyright 2017 6WIND S.A. + +//! Functions in this module are used to handle eBPF programs with a higher level representation, +//! for example to disassemble the code into a human-readable format. + +use alloc::{ + format, + string::{String, ToString}, + vec, + vec::Vec, +}; + +use log::warn; + +use crate::ebpf; + +#[inline] +fn alu_imm_str(name: &str, insn: &ebpf::Insn) -> String { + format!("{name} r{}, {:#x}", insn.dst, insn.imm) +} + +#[inline] +fn alu_reg_str(name: &str, insn: &ebpf::Insn) -> String { + format!("{name} r{}, r{}", insn.dst, insn.src) +} + +#[inline] +fn byteswap_str(name: &str, insn: &ebpf::Insn) -> String { + match insn.imm { + 16 | 32 | 64 => {} + _ => warn!("[Disassembler] Warning: Invalid offset value for {name} insn"), + } + format!("{name}{} r{}", insn.imm, insn.dst) +} + +#[inline] +fn ld_st_imm_str(name: &str, insn: &ebpf::Insn) -> String { + if insn.off >= 0 { + format!("{name} [r{}+{:#x}], {:#x}", insn.dst, insn.off, insn.imm) + } else { + format!( + "{name} [r{}-{:#x}], {:#x}", + insn.dst, + -(insn.off as isize), + insn.imm + ) + } +} + +#[inline] +fn ld_reg_str(name: &str, insn: &ebpf::Insn) -> String { + if insn.off >= 0 { + format!("{name} r{}, [r{}+{:#x}]", insn.dst, insn.src, insn.off) + } else { + format!( + "{name} r{}, [r{}-{:#x}]", + insn.dst, + insn.src, + -(insn.off as isize) + ) + } +} + +#[inline] +fn st_reg_str(name: &str, insn: &ebpf::Insn) -> String { + if insn.off >= 0 { + format!("{name} [r{}+{:#x}], r{}", insn.dst, insn.off, insn.src) + } else { + format!( + "{name} [r{}-{:#x}], r{}", + insn.dst, + -(insn.off as isize), + insn.src + ) + } +} + +#[inline] +fn ldabs_str(name: &str, insn: &ebpf::Insn) -> String { + format!("{name} {:#x}", insn.imm) +} + +#[inline] +fn ldind_str(name: &str, insn: &ebpf::Insn) -> String { + format!("{name} r{}, {:#x}", insn.src, insn.imm) +} + +#[inline] +fn jmp_imm_str(name: &str, insn: &ebpf::Insn) -> String { + if insn.off >= 0 { + format!("{name} r{}, {:#x}, +{:#x}", insn.dst, insn.imm, insn.off) + } else { + format!( + "{name} r{}, {:#x}, -{:#x}", + insn.dst, + insn.imm, + -(insn.off as isize) + ) + } +} + +#[inline] +fn jmp_reg_str(name: &str, insn: &ebpf::Insn) -> String { + if insn.off >= 0 { + format!("{name} r{}, r{}, +{:#x}", insn.dst, insn.src, insn.off) + } else { + format!( + "{name} r{}, r{}, -{:#x}", + insn.dst, + insn.src, + -(insn.off as isize) + ) + } +} + +/// High-level representation of an eBPF instruction. +/// +/// In addition to standard operation code and various operand, this struct has the following +/// properties: +/// +/// * It stores a name, corresponding to a mnemonic for the operation code. +/// * It also stores a description, which is a mnemonic for the full instruction, using the actual +/// values of the relevant operands, and that can be used for disassembling the eBPF program for +/// example. +/// * Immediate values are stored in an `i64` instead of a traditional i32, in order to merge the +/// two parts of (otherwise double-length) `LD_DW_IMM` instructions. +/// +/// See for the Linux kernel +/// documentation about eBPF, or for a +/// more concise version. +#[derive(Debug, PartialEq, Eq)] +pub struct HLInsn { + /// Operation code. + pub opc: u8, + /// Name (mnemonic). This name is not canon. + pub name: String, + /// Description of the instruction. This is not canon. + pub desc: String, + /// Destination register operand. + pub dst: u8, + /// Source register operand. + pub src: u8, + /// Offset operand. + pub off: i16, + /// Immediate value operand. For `LD_DW_IMM` instructions, contains the whole value merged from + /// the two 8-bytes parts of the instruction. + pub imm: i64, +} + +/// Return a vector of `struct HLInsn` built from an eBPF program. +/// +/// This is made public to provide a way to manipulate a program as a vector of instructions, in a +/// high-level format, for example for dumping the program instruction after instruction with a +/// custom format. +/// +/// Note that the two parts of `LD_DW_IMM` instructions (that have the size of two standard +/// instructions) are considered as making a single immediate value. As a consequence, the number +/// of instructions stored in the vector may not be equal to the size in bytes of the program +/// divided by the length of an instructions. +/// +/// To do so, the immediate value operand is stored as an `i64` instead as an i32, so be careful +/// when you use it (see example `examples/to_json.rs`). +/// +/// This is to oppose to `ebpf::to_insn_vec()` function, that treats instructions on a low-level +/// ground and do not merge the parts of `LD_DW_IMM`. Also, the version in `ebpf` module does not +/// use names or descriptions when storing the instructions. +/// +/// # Examples +/// +/// ``` +/// use rbpf::disassembler; +/// +/// let prog = &[ +/// 0x18, 0x00, 0x00, 0x00, 0x88, 0x77, 0x66, 0x55, +/// 0x00, 0x00, 0x00, 0x00, 0x44, 0x33, 0x22, 0x11, +/// 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 +/// ]; +/// +/// let v = disassembler::to_insn_vec(prog); +/// assert_eq!(v, vec![ +/// disassembler::HLInsn { +/// opc: 0x18, +/// name: "lddw".to_string(), +/// desc: "lddw r0, 0x1122334455667788".to_string(), +/// dst: 0, +/// src: 0, +/// off: 0, +/// imm: 0x1122334455667788 +/// }, +/// disassembler::HLInsn { +/// opc: 0x95, +/// name: "exit".to_string(), +/// desc: "exit".to_string(), +/// dst: 0, +/// src: 0, +/// off: 0, +/// imm: 0 +/// }, +/// ]); +/// ``` +pub fn to_insn_vec(prog: &[u8]) -> Vec { + if prog.len() % ebpf::INSN_SIZE != 0 { + panic!( + "[Disassembler] Error: eBPF program length must be a multiple of {:?} octets", + ebpf::INSN_SIZE + ); + } + if prog.is_empty() { + return vec![]; + } + + let mut res = vec![]; + let mut insn_ptr: usize = 0; + + while insn_ptr * ebpf::INSN_SIZE < prog.len() { + let insn = ebpf::get_insn(prog, insn_ptr); + + let name; + let desc; + let mut imm = insn.imm as i64; + match insn.opc { + // BPF_LD class + ebpf::LD_ABS_B => { + name = "ldabsb"; + desc = ldabs_str(name, &insn); + } + ebpf::LD_ABS_H => { + name = "ldabsh"; + desc = ldabs_str(name, &insn); + } + ebpf::LD_ABS_W => { + name = "ldabsw"; + desc = ldabs_str(name, &insn); + } + ebpf::LD_ABS_DW => { + name = "ldabsdw"; + desc = ldabs_str(name, &insn); + } + ebpf::LD_IND_B => { + name = "ldindb"; + desc = ldind_str(name, &insn); + } + ebpf::LD_IND_H => { + name = "ldindh"; + desc = ldind_str(name, &insn); + } + ebpf::LD_IND_W => { + name = "ldindw"; + desc = ldind_str(name, &insn); + } + ebpf::LD_IND_DW => { + name = "ldinddw"; + desc = ldind_str(name, &insn); + } + + ebpf::LD_DW_IMM => { + insn_ptr += 1; + let next_insn = ebpf::get_insn(prog, insn_ptr); + imm = ((insn.imm as u32) as u64 + ((next_insn.imm as u64) << 32)) as i64; + name = "lddw"; + desc = format!("{name} r{:}, {imm:#x}", insn.dst); + } + + // BPF_LDX class + ebpf::LD_B_REG => { + name = "ldxb"; + desc = ld_reg_str(name, &insn); + } + ebpf::LD_H_REG => { + name = "ldxh"; + desc = ld_reg_str(name, &insn); + } + ebpf::LD_W_REG => { + name = "ldxw"; + desc = ld_reg_str(name, &insn); + } + ebpf::LD_DW_REG => { + name = "ldxdw"; + desc = ld_reg_str(name, &insn); + } + + // BPF_ST class + ebpf::ST_B_IMM => { + name = "stb"; + desc = ld_st_imm_str(name, &insn); + } + ebpf::ST_H_IMM => { + name = "sth"; + desc = ld_st_imm_str(name, &insn); + } + ebpf::ST_W_IMM => { + name = "stw"; + desc = ld_st_imm_str(name, &insn); + } + ebpf::ST_DW_IMM => { + name = "stdw"; + desc = ld_st_imm_str(name, &insn); + } + + // BPF_STX class + ebpf::ST_B_REG => { + name = "stxb"; + desc = st_reg_str(name, &insn); + } + ebpf::ST_H_REG => { + name = "stxh"; + desc = st_reg_str(name, &insn); + } + ebpf::ST_W_REG => { + name = "stxw"; + desc = st_reg_str(name, &insn); + } + ebpf::ST_DW_REG => { + name = "stxdw"; + desc = st_reg_str(name, &insn); + } + ebpf::ST_W_XADD => { + name = "stxxaddw"; + desc = st_reg_str(name, &insn); + } + ebpf::ST_DW_XADD => { + name = "stxxadddw"; + desc = st_reg_str(name, &insn); + } + + // BPF_ALU class + ebpf::ADD32_IMM => { + name = "add32"; + desc = alu_imm_str(name, &insn); + } + ebpf::ADD32_REG => { + name = "add32"; + desc = alu_reg_str(name, &insn); + } + ebpf::SUB32_IMM => { + name = "sub32"; + desc = alu_imm_str(name, &insn); + } + ebpf::SUB32_REG => { + name = "sub32"; + desc = alu_reg_str(name, &insn); + } + ebpf::MUL32_IMM => { + name = "mul32"; + desc = alu_imm_str(name, &insn); + } + ebpf::MUL32_REG => { + name = "mul32"; + desc = alu_reg_str(name, &insn); + } + ebpf::DIV32_IMM => { + name = "div32"; + desc = alu_imm_str(name, &insn); + } + ebpf::DIV32_REG => { + name = "div32"; + desc = alu_reg_str(name, &insn); + } + ebpf::OR32_IMM => { + name = "or32"; + desc = alu_imm_str(name, &insn); + } + ebpf::OR32_REG => { + name = "or32"; + desc = alu_reg_str(name, &insn); + } + ebpf::AND32_IMM => { + name = "and32"; + desc = alu_imm_str(name, &insn); + } + ebpf::AND32_REG => { + name = "and32"; + desc = alu_reg_str(name, &insn); + } + ebpf::LSH32_IMM => { + name = "lsh32"; + desc = alu_imm_str(name, &insn); + } + ebpf::LSH32_REG => { + name = "lsh32"; + desc = alu_reg_str(name, &insn); + } + ebpf::RSH32_IMM => { + name = "rsh32"; + desc = alu_imm_str(name, &insn); + } + ebpf::RSH32_REG => { + name = "rsh32"; + desc = alu_reg_str(name, &insn); + } + ebpf::NEG32 => { + name = "neg32"; + desc = format!("{name} r{:}", insn.dst); + } + ebpf::MOD32_IMM => { + name = "mod32"; + desc = alu_imm_str(name, &insn); + } + ebpf::MOD32_REG => { + name = "mod32"; + desc = alu_reg_str(name, &insn); + } + ebpf::XOR32_IMM => { + name = "xor32"; + desc = alu_imm_str(name, &insn); + } + ebpf::XOR32_REG => { + name = "xor32"; + desc = alu_reg_str(name, &insn); + } + ebpf::MOV32_IMM => { + name = "mov32"; + desc = alu_imm_str(name, &insn); + } + ebpf::MOV32_REG => { + name = "mov32"; + desc = alu_reg_str(name, &insn); + } + ebpf::ARSH32_IMM => { + name = "arsh32"; + desc = alu_imm_str(name, &insn); + } + ebpf::ARSH32_REG => { + name = "arsh32"; + desc = alu_reg_str(name, &insn); + } + ebpf::LE => { + name = "le"; + desc = byteswap_str(name, &insn); + } + ebpf::BE => { + name = "be"; + desc = byteswap_str(name, &insn); + } + + // BPF_ALU64 class + ebpf::ADD64_IMM => { + name = "add64"; + desc = alu_imm_str(name, &insn); + } + ebpf::ADD64_REG => { + name = "add64"; + desc = alu_reg_str(name, &insn); + } + ebpf::SUB64_IMM => { + name = "sub64"; + desc = alu_imm_str(name, &insn); + } + ebpf::SUB64_REG => { + name = "sub64"; + desc = alu_reg_str(name, &insn); + } + ebpf::MUL64_IMM => { + name = "mul64"; + desc = alu_imm_str(name, &insn); + } + ebpf::MUL64_REG => { + name = "mul64"; + desc = alu_reg_str(name, &insn); + } + ebpf::DIV64_IMM => { + name = "div64"; + desc = alu_imm_str(name, &insn); + } + ebpf::DIV64_REG => { + name = "div64"; + desc = alu_reg_str(name, &insn); + } + ebpf::OR64_IMM => { + name = "or64"; + desc = alu_imm_str(name, &insn); + } + ebpf::OR64_REG => { + name = "or64"; + desc = alu_reg_str(name, &insn); + } + ebpf::AND64_IMM => { + name = "and64"; + desc = alu_imm_str(name, &insn); + } + ebpf::AND64_REG => { + name = "and64"; + desc = alu_reg_str(name, &insn); + } + ebpf::LSH64_IMM => { + name = "lsh64"; + desc = alu_imm_str(name, &insn); + } + ebpf::LSH64_REG => { + name = "lsh64"; + desc = alu_reg_str(name, &insn); + } + ebpf::RSH64_IMM => { + name = "rsh64"; + desc = alu_imm_str(name, &insn); + } + ebpf::RSH64_REG => { + name = "rsh64"; + desc = alu_reg_str(name, &insn); + } + ebpf::NEG64 => { + name = "neg64"; + desc = format!("{name} r{:}", insn.dst); + } + ebpf::MOD64_IMM => { + name = "mod64"; + desc = alu_imm_str(name, &insn); + } + ebpf::MOD64_REG => { + name = "mod64"; + desc = alu_reg_str(name, &insn); + } + ebpf::XOR64_IMM => { + name = "xor64"; + desc = alu_imm_str(name, &insn); + } + ebpf::XOR64_REG => { + name = "xor64"; + desc = alu_reg_str(name, &insn); + } + ebpf::MOV64_IMM => { + name = "mov64"; + desc = alu_imm_str(name, &insn); + } + ebpf::MOV64_REG => { + name = "mov64"; + desc = alu_reg_str(name, &insn); + } + ebpf::ARSH64_IMM => { + name = "arsh64"; + desc = alu_imm_str(name, &insn); + } + ebpf::ARSH64_REG => { + name = "arsh64"; + desc = alu_reg_str(name, &insn); + } + + // BPF_JMP class + ebpf::JA => { + name = "ja"; + desc = if insn.off >= 0 { + format!("{name} +{:#x}", insn.off) + } else { + format!("{name} -{:#x}", -insn.off) + } + } + ebpf::JEQ_IMM => { + name = "jeq"; + desc = jmp_imm_str(name, &insn); + } + ebpf::JEQ_REG => { + name = "jeq"; + desc = jmp_reg_str(name, &insn); + } + ebpf::JGT_IMM => { + name = "jgt"; + desc = jmp_imm_str(name, &insn); + } + ebpf::JGT_REG => { + name = "jgt"; + desc = jmp_reg_str(name, &insn); + } + ebpf::JGE_IMM => { + name = "jge"; + desc = jmp_imm_str(name, &insn); + } + ebpf::JGE_REG => { + name = "jge"; + desc = jmp_reg_str(name, &insn); + } + ebpf::JLT_IMM => { + name = "jlt"; + desc = jmp_imm_str(name, &insn); + } + ebpf::JLT_REG => { + name = "jlt"; + desc = jmp_reg_str(name, &insn); + } + ebpf::JLE_IMM => { + name = "jle"; + desc = jmp_imm_str(name, &insn); + } + ebpf::JLE_REG => { + name = "jle"; + desc = jmp_reg_str(name, &insn); + } + ebpf::JSET_IMM => { + name = "jset"; + desc = jmp_imm_str(name, &insn); + } + ebpf::JSET_REG => { + name = "jset"; + desc = jmp_reg_str(name, &insn); + } + ebpf::JNE_IMM => { + name = "jne"; + desc = jmp_imm_str(name, &insn); + } + ebpf::JNE_REG => { + name = "jne"; + desc = jmp_reg_str(name, &insn); + } + ebpf::JSGT_IMM => { + name = "jsgt"; + desc = jmp_imm_str(name, &insn); + } + ebpf::JSGT_REG => { + name = "jsgt"; + desc = jmp_reg_str(name, &insn); + } + ebpf::JSGE_IMM => { + name = "jsge"; + desc = jmp_imm_str(name, &insn); + } + ebpf::JSGE_REG => { + name = "jsge"; + desc = jmp_reg_str(name, &insn); + } + ebpf::JSLT_IMM => { + name = "jslt"; + desc = jmp_imm_str(name, &insn); + } + ebpf::JSLT_REG => { + name = "jslt"; + desc = jmp_reg_str(name, &insn); + } + ebpf::JSLE_IMM => { + name = "jsle"; + desc = jmp_imm_str(name, &insn); + } + ebpf::JSLE_REG => { + name = "jsle"; + desc = jmp_reg_str(name, &insn); + } + ebpf::CALL => { + name = "call"; + desc = format!("{name} {:#x}", insn.imm); + } + ebpf::TAIL_CALL => { + name = "tail_call"; + desc = name.to_string(); + } + ebpf::EXIT => { + name = "exit"; + desc = name.to_string(); + } + + // BPF_JMP32 class + ebpf::JEQ_IMM32 => { + name = "jeq32"; + desc = jmp_imm_str(name, &insn); + } + ebpf::JEQ_REG32 => { + name = "jeq32"; + desc = jmp_reg_str(name, &insn); + } + ebpf::JGT_IMM32 => { + name = "jgt32"; + desc = jmp_imm_str(name, &insn); + } + ebpf::JGT_REG32 => { + name = "jgt32"; + desc = jmp_reg_str(name, &insn); + } + ebpf::JGE_IMM32 => { + name = "jge32"; + desc = jmp_imm_str(name, &insn); + } + ebpf::JGE_REG32 => { + name = "jge32"; + desc = jmp_reg_str(name, &insn); + } + ebpf::JLT_IMM32 => { + name = "jlt32"; + desc = jmp_imm_str(name, &insn); + } + ebpf::JLT_REG32 => { + name = "jlt32"; + desc = jmp_reg_str(name, &insn); + } + ebpf::JLE_IMM32 => { + name = "jle32"; + desc = jmp_imm_str(name, &insn); + } + ebpf::JLE_REG32 => { + name = "jle32"; + desc = jmp_reg_str(name, &insn); + } + ebpf::JSET_IMM32 => { + name = "jset32"; + desc = jmp_imm_str(name, &insn); + } + ebpf::JSET_REG32 => { + name = "jset32"; + desc = jmp_reg_str(name, &insn); + } + ebpf::JNE_IMM32 => { + name = "jne32"; + desc = jmp_imm_str(name, &insn); + } + ebpf::JNE_REG32 => { + name = "jne32"; + desc = jmp_reg_str(name, &insn); + } + ebpf::JSGT_IMM32 => { + name = "jsgt32"; + desc = jmp_imm_str(name, &insn); + } + ebpf::JSGT_REG32 => { + name = "jsgt32"; + desc = jmp_reg_str(name, &insn); + } + ebpf::JSGE_IMM32 => { + name = "jsge32"; + desc = jmp_imm_str(name, &insn); + } + ebpf::JSGE_REG32 => { + name = "jsge32"; + desc = jmp_reg_str(name, &insn); + } + ebpf::JSLT_IMM32 => { + name = "jslt32"; + desc = jmp_imm_str(name, &insn); + } + ebpf::JSLT_REG32 => { + name = "jslt32"; + desc = jmp_reg_str(name, &insn); + } + ebpf::JSLE_IMM32 => { + name = "jsle32"; + desc = jmp_imm_str(name, &insn); + } + ebpf::JSLE_REG32 => { + name = "jsle32"; + desc = jmp_reg_str(name, &insn); + } + + _ => { + panic!( + "[Disassembler] Error: unknown eBPF opcode {:#2x} (insn #{:?})", + insn.opc, insn_ptr + ); + } + }; + + let hl_insn = HLInsn { + opc: insn.opc, + name: name.to_string(), + desc, + dst: insn.dst, + src: insn.src, + off: insn.off, + imm, + }; + + res.push(hl_insn); + + insn_ptr += 1; + } + res +} + +/// Disassemble an eBPF program into human-readable instructions and prints it to standard output. +/// +/// The program is not checked for errors or inconsistencies. +/// +/// # Examples +/// +/// ``` +/// use rbpf::disassembler; +/// let prog = &[ +/// 0x07, 0x01, 0x00, 0x00, 0x05, 0x06, 0x00, 0x00, +/// 0xb7, 0x02, 0x00, 0x00, 0x32, 0x00, 0x00, 0x00, +/// 0xbf, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +/// 0xdc, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, +/// 0x87, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +/// 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 +/// ]; +/// disassembler::disassemble(prog); +/// # // "\nadd64 r1, 0x605\nmov64 r2, 0x32\nmov64 r1, r0\nbe16 r0\nneg64 r2\nexit" +/// ``` +/// +/// This will produce the following output: +/// +/// ```test +/// add64 r1, 0x605 +/// mov64 r2, 0x32 +/// mov64 r1, r0 +/// be16 r0 +/// neg64 r2 +/// exit +/// ``` +pub fn disassemble(prog: &[u8]) { + #[cfg(feature = "std")] + { + for insn in to_insn_vec(prog) { + println!("{}", insn.desc); + } + } + #[cfg(not(feature = "std"))] + { + for insn in to_insn_vec(prog) { + log::info!("{}", insn.desc); + } + } +} diff --git a/kernel/crates/rbpf/src/ebpf.rs b/kernel/crates/rbpf/src/ebpf.rs new file mode 100644 index 000000000..33018b06a --- /dev/null +++ b/kernel/crates/rbpf/src/ebpf.rs @@ -0,0 +1,635 @@ +// SPDX-License-Identifier: (Apache-2.0 OR MIT) +// Copyright 2016 6WIND S.A. + +//! This module contains all the definitions related to eBPF, and some functions permitting to +//! manipulate eBPF instructions. +//! +//! The number of bytes in an instruction, the maximum number of instructions in a program, and +//! also all operation codes are defined here as constants. +//! +//! The structure for an instruction used by this crate, as well as the function to extract it from +//! a program, is also defined in the module. +//! +//! To learn more about these instructions, see the Linux kernel documentation: +//! , or for a shorter version of +//! the list of the operation codes: + +use alloc::{vec, vec::Vec}; + +use byteorder::{ByteOrder, LittleEndian}; + +/// The maximum call depth is 8 +pub const RBPF_MAX_CALL_DEPTH: usize = 8; + +/// Maximum number of instructions in an eBPF program. +pub const PROG_MAX_INSNS: usize = 1000000; +/// Size of an eBPF instructions, in bytes. +pub const INSN_SIZE: usize = 8; +/// Maximum size of an eBPF program, in bytes. +pub const PROG_MAX_SIZE: usize = PROG_MAX_INSNS * INSN_SIZE; +/// Stack for the eBPF stack, in bytes. +pub const STACK_SIZE: usize = 512; + +// eBPF op codes. +// See also https://www.kernel.org/doc/Documentation/networking/filter.txt + +// Three least significant bits are operation class: +/// BPF operation class: load from immediate. +pub const BPF_LD: u8 = 0x00; +/// BPF operation class: load from register. +pub const BPF_LDX: u8 = 0x01; +/// BPF operation class: store immediate. +pub const BPF_ST: u8 = 0x02; +/// BPF operation class: store value from register. +pub const BPF_STX: u8 = 0x03; +/// BPF operation class: 32 bits arithmetic operation. +pub const BPF_ALU: u8 = 0x04; +/// BPF operation class: jump (64-bit wide operands for comparisons). +pub const BPF_JMP: u8 = 0x05; +/// BPF operation class: jump (32-bit wide operands for comparisons). +pub const BPF_JMP32: u8 = 0x06; +// [ class 6 unused, reserved for future use ] +/// BPF operation class: 64 bits arithmetic operation. +pub const BPF_ALU64: u8 = 0x07; + +// For load and store instructions: +// +------------+--------+------------+ +// | 3 bits | 2 bits | 3 bits | +// | mode | size | insn class | +// +------------+--------+------------+ +// (MSB) (LSB) + +// Size modifiers: +/// BPF size modifier: word (4 bytes). +pub const BPF_W: u8 = 0x00; +/// BPF size modifier: half-word (2 bytes). +pub const BPF_H: u8 = 0x08; +/// BPF size modifier: byte (1 byte). +pub const BPF_B: u8 = 0x10; +/// BPF size modifier: double word (8 bytes). +pub const BPF_DW: u8 = 0x18; + +// Mode modifiers: +/// BPF mode modifier: immediate value. +pub const BPF_IMM: u8 = 0x00; +/// BPF mode modifier: absolute load. +pub const BPF_ABS: u8 = 0x20; +/// BPF mode modifier: indirect load. +pub const BPF_IND: u8 = 0x40; +/// BPF mode modifier: load from / store to memory. +pub const BPF_MEM: u8 = 0x60; +// [ 0x80 reserved ] +// [ 0xa0 reserved ] +/// BPF mode modifier: exclusive add. +pub const BPF_XADD: u8 = 0xc0; + +// For arithmetic (BPF_ALU/BPF_ALU64) and jump (BPF_JMP) instructions: +// +----------------+--------+--------+ +// | 4 bits |1 b.| 3 bits | +// | operation code | src| insn class | +// +----------------+----+------------+ +// (MSB) (LSB) + +// Source modifiers: +/// BPF source operand modifier: 32-bit immediate value. +pub const BPF_K: u8 = 0x00; +/// BPF source operand modifier: `src` register. +pub const BPF_X: u8 = 0x08; + +// Operation codes -- BPF_ALU or BPF_ALU64 classes: +/// BPF ALU/ALU64 operation code: addition. +pub const BPF_ADD: u8 = 0x00; +/// BPF ALU/ALU64 operation code: subtraction. +pub const BPF_SUB: u8 = 0x10; +/// BPF ALU/ALU64 operation code: multiplication. +pub const BPF_MUL: u8 = 0x20; +/// BPF ALU/ALU64 operation code: division. +pub const BPF_DIV: u8 = 0x30; +/// BPF ALU/ALU64 operation code: or. +pub const BPF_OR: u8 = 0x40; +/// BPF ALU/ALU64 operation code: and. +pub const BPF_AND: u8 = 0x50; +/// BPF ALU/ALU64 operation code: left shift. +pub const BPF_LSH: u8 = 0x60; +/// BPF ALU/ALU64 operation code: right shift. +pub const BPF_RSH: u8 = 0x70; +/// BPF ALU/ALU64 operation code: negation. +pub const BPF_NEG: u8 = 0x80; +/// BPF ALU/ALU64 operation code: modulus. +pub const BPF_MOD: u8 = 0x90; +/// BPF ALU/ALU64 operation code: exclusive or. +pub const BPF_XOR: u8 = 0xa0; +/// BPF ALU/ALU64 operation code: move. +pub const BPF_MOV: u8 = 0xb0; +/// BPF ALU/ALU64 operation code: sign extending right shift. +pub const BPF_ARSH: u8 = 0xc0; +/// BPF ALU/ALU64 operation code: endianness conversion. +pub const BPF_END: u8 = 0xd0; + +// Operation codes -- BPF_JMP or BPF_JMP32 classes: +/// BPF JMP operation code: jump. +pub const BPF_JA: u8 = 0x00; +/// BPF JMP operation code: jump if equal. +pub const BPF_JEQ: u8 = 0x10; +/// BPF JMP operation code: jump if greater than. +pub const BPF_JGT: u8 = 0x20; +/// BPF JMP operation code: jump if greater or equal. +pub const BPF_JGE: u8 = 0x30; +/// BPF JMP operation code: jump if `src` & `reg`. +pub const BPF_JSET: u8 = 0x40; +/// BPF JMP operation code: jump if not equal. +pub const BPF_JNE: u8 = 0x50; +/// BPF JMP operation code: jump if greater than (signed). +pub const BPF_JSGT: u8 = 0x60; +/// BPF JMP operation code: jump if greater or equal (signed). +pub const BPF_JSGE: u8 = 0x70; +/// BPF JMP operation code: helper function call. +pub const BPF_CALL: u8 = 0x80; +/// BPF JMP operation code: return from program. +pub const BPF_EXIT: u8 = 0x90; +/// BPF JMP operation code: jump if lower than. +pub const BPF_JLT: u8 = 0xa0; +/// BPF JMP operation code: jump if lower or equal. +pub const BPF_JLE: u8 = 0xb0; +/// BPF JMP operation code: jump if lower than (signed). +pub const BPF_JSLT: u8 = 0xc0; +/// BPF JMP operation code: jump if lower or equal (signed). +pub const BPF_JSLE: u8 = 0xd0; + +// Op codes +// (Following operation names are not “official”, but may be proper to rbpf; Linux kernel only +// combines above flags and does not attribute a name per operation.) + +/// BPF opcode: `ldabsb src, dst, imm`. +pub const LD_ABS_B: u8 = BPF_LD | BPF_ABS | BPF_B; +/// BPF opcode: `ldabsh src, dst, imm`. +pub const LD_ABS_H: u8 = BPF_LD | BPF_ABS | BPF_H; +/// BPF opcode: `ldabsw src, dst, imm`. +pub const LD_ABS_W: u8 = BPF_LD | BPF_ABS | BPF_W; +/// BPF opcode: `ldabsdw src, dst, imm`. +pub const LD_ABS_DW: u8 = BPF_LD | BPF_ABS | BPF_DW; +/// BPF opcode: `ldindb src, dst, imm`. +pub const LD_IND_B: u8 = BPF_LD | BPF_IND | BPF_B; +/// BPF opcode: `ldindh src, dst, imm`. +pub const LD_IND_H: u8 = BPF_LD | BPF_IND | BPF_H; +/// BPF opcode: `ldindw src, dst, imm`. +pub const LD_IND_W: u8 = BPF_LD | BPF_IND | BPF_W; +/// BPF opcode: `ldinddw src, dst, imm`. +pub const LD_IND_DW: u8 = BPF_LD | BPF_IND | BPF_DW; + +#[allow(unknown_lints)] +#[allow(clippy::eq_op)] +/// BPF opcode: `lddw dst, imm` /// `dst = imm`. +pub const LD_DW_IMM: u8 = BPF_LD | BPF_IMM | BPF_DW; +/// BPF opcode: `ldxb dst, [src + off]` /// `dst = (src + off) as u8`. +pub const LD_B_REG: u8 = BPF_LDX | BPF_MEM | BPF_B; +/// BPF opcode: `ldxh dst, [src + off]` /// `dst = (src + off) as u16`. +pub const LD_H_REG: u8 = BPF_LDX | BPF_MEM | BPF_H; +/// BPF opcode: `ldxw dst, [src + off]` /// `dst = (src + off) as u32`. +pub const LD_W_REG: u8 = BPF_LDX | BPF_MEM | BPF_W; +/// BPF opcode: `ldxdw dst, [src + off]` /// `dst = (src + off) as u64`. +pub const LD_DW_REG: u8 = BPF_LDX | BPF_MEM | BPF_DW; +/// BPF opcode: `stb [dst + off], imm` /// `(dst + offset) as u8 = imm`. +pub const ST_B_IMM: u8 = BPF_ST | BPF_MEM | BPF_B; +/// BPF opcode: `sth [dst + off], imm` /// `(dst + offset) as u16 = imm`. +pub const ST_H_IMM: u8 = BPF_ST | BPF_MEM | BPF_H; +/// BPF opcode: `stw [dst + off], imm` /// `(dst + offset) as u32 = imm`. +pub const ST_W_IMM: u8 = BPF_ST | BPF_MEM | BPF_W; +/// BPF opcode: `stdw [dst + off], imm` /// `(dst + offset) as u64 = imm`. +pub const ST_DW_IMM: u8 = BPF_ST | BPF_MEM | BPF_DW; +/// BPF opcode: `stxb [dst + off], src` /// `(dst + offset) as u8 = src`. +pub const ST_B_REG: u8 = BPF_STX | BPF_MEM | BPF_B; +/// BPF opcode: `stxh [dst + off], src` /// `(dst + offset) as u16 = src`. +pub const ST_H_REG: u8 = BPF_STX | BPF_MEM | BPF_H; +/// BPF opcode: `stxw [dst + off], src` /// `(dst + offset) as u32 = src`. +pub const ST_W_REG: u8 = BPF_STX | BPF_MEM | BPF_W; +/// BPF opcode: `stxdw [dst + off], src` /// `(dst + offset) as u64 = src`. +pub const ST_DW_REG: u8 = BPF_STX | BPF_MEM | BPF_DW; + +/// BPF opcode: `stxxaddw [dst + off], src`. +pub const ST_W_XADD: u8 = BPF_STX | BPF_XADD | BPF_W; +/// BPF opcode: `stxxadddw [dst + off], src`. +pub const ST_DW_XADD: u8 = BPF_STX | BPF_XADD | BPF_DW; + +/// BPF opcode: `add32 dst, imm` /// `dst += imm`. +pub const ADD32_IMM: u8 = BPF_ALU | BPF_K | BPF_ADD; +/// BPF opcode: `add32 dst, src` /// `dst += src`. +pub const ADD32_REG: u8 = BPF_ALU | BPF_X | BPF_ADD; +/// BPF opcode: `sub32 dst, imm` /// `dst -= imm`. +pub const SUB32_IMM: u8 = BPF_ALU | BPF_K | BPF_SUB; +/// BPF opcode: `sub32 dst, src` /// `dst -= src`. +pub const SUB32_REG: u8 = BPF_ALU | BPF_X | BPF_SUB; +/// BPF opcode: `mul32 dst, imm` /// `dst *= imm`. +pub const MUL32_IMM: u8 = BPF_ALU | BPF_K | BPF_MUL; +/// BPF opcode: `mul32 dst, src` /// `dst *= src`. +pub const MUL32_REG: u8 = BPF_ALU | BPF_X | BPF_MUL; +/// BPF opcode: `div32 dst, imm` /// `dst /= imm`. +pub const DIV32_IMM: u8 = BPF_ALU | BPF_K | BPF_DIV; +/// BPF opcode: `div32 dst, src` /// `dst /= src`. +pub const DIV32_REG: u8 = BPF_ALU | BPF_X | BPF_DIV; +/// BPF opcode: `or32 dst, imm` /// `dst |= imm`. +pub const OR32_IMM: u8 = BPF_ALU | BPF_K | BPF_OR; +/// BPF opcode: `or32 dst, src` /// `dst |= src`. +pub const OR32_REG: u8 = BPF_ALU | BPF_X | BPF_OR; +/// BPF opcode: `and32 dst, imm` /// `dst &= imm`. +pub const AND32_IMM: u8 = BPF_ALU | BPF_K | BPF_AND; +/// BPF opcode: `and32 dst, src` /// `dst &= src`. +pub const AND32_REG: u8 = BPF_ALU | BPF_X | BPF_AND; +/// BPF opcode: `lsh32 dst, imm` /// `dst <<= imm`. +pub const LSH32_IMM: u8 = BPF_ALU | BPF_K | BPF_LSH; +/// BPF opcode: `lsh32 dst, src` /// `dst <<= src`. +pub const LSH32_REG: u8 = BPF_ALU | BPF_X | BPF_LSH; +/// BPF opcode: `rsh32 dst, imm` /// `dst >>= imm`. +pub const RSH32_IMM: u8 = BPF_ALU | BPF_K | BPF_RSH; +/// BPF opcode: `rsh32 dst, src` /// `dst >>= src`. +pub const RSH32_REG: u8 = BPF_ALU | BPF_X | BPF_RSH; +/// BPF opcode: `neg32 dst` /// `dst = -dst`. +pub const NEG32: u8 = BPF_ALU | BPF_NEG; +/// BPF opcode: `mod32 dst, imm` /// `dst %= imm`. +pub const MOD32_IMM: u8 = BPF_ALU | BPF_K | BPF_MOD; +/// BPF opcode: `mod32 dst, src` /// `dst %= src`. +pub const MOD32_REG: u8 = BPF_ALU | BPF_X | BPF_MOD; +/// BPF opcode: `xor32 dst, imm` /// `dst ^= imm`. +pub const XOR32_IMM: u8 = BPF_ALU | BPF_K | BPF_XOR; +/// BPF opcode: `xor32 dst, src` /// `dst ^= src`. +pub const XOR32_REG: u8 = BPF_ALU | BPF_X | BPF_XOR; +/// BPF opcode: `mov32 dst, imm` /// `dst = imm`. +pub const MOV32_IMM: u8 = BPF_ALU | BPF_K | BPF_MOV; +/// BPF opcode: `mov32 dst, src` /// `dst = src`. +pub const MOV32_REG: u8 = BPF_ALU | BPF_X | BPF_MOV; +/// BPF opcode: `arsh32 dst, imm` /// `dst >>= imm (arithmetic)`. +/// +/// +pub const ARSH32_IMM: u8 = BPF_ALU | BPF_K | BPF_ARSH; +/// BPF opcode: `arsh32 dst, src` /// `dst >>= src (arithmetic)`. +/// +/// +pub const ARSH32_REG: u8 = BPF_ALU | BPF_X | BPF_ARSH; + +/// BPF opcode: `le dst` /// `dst = htole(dst), with imm in {16, 32, 64}`. +pub const LE: u8 = BPF_ALU | BPF_K | BPF_END; +/// BPF opcode: `be dst` /// `dst = htobe(dst), with imm in {16, 32, 64}`. +pub const BE: u8 = BPF_ALU | BPF_X | BPF_END; + +/// BPF opcode: `add64 dst, imm` /// `dst += imm`. +pub const ADD64_IMM: u8 = BPF_ALU64 | BPF_K | BPF_ADD; +/// BPF opcode: `add64 dst, src` /// `dst += src`. +pub const ADD64_REG: u8 = BPF_ALU64 | BPF_X | BPF_ADD; +/// BPF opcode: `sub64 dst, imm` /// `dst -= imm`. +pub const SUB64_IMM: u8 = BPF_ALU64 | BPF_K | BPF_SUB; +/// BPF opcode: `sub64 dst, src` /// `dst -= src`. +pub const SUB64_REG: u8 = BPF_ALU64 | BPF_X | BPF_SUB; +/// BPF opcode: `div64 dst, imm` /// `dst /= imm`. +pub const MUL64_IMM: u8 = BPF_ALU64 | BPF_K | BPF_MUL; +/// BPF opcode: `div64 dst, src` /// `dst /= src`. +pub const MUL64_REG: u8 = BPF_ALU64 | BPF_X | BPF_MUL; +/// BPF opcode: `div64 dst, imm` /// `dst /= imm`. +pub const DIV64_IMM: u8 = BPF_ALU64 | BPF_K | BPF_DIV; +/// BPF opcode: `div64 dst, src` /// `dst /= src`. +pub const DIV64_REG: u8 = BPF_ALU64 | BPF_X | BPF_DIV; +/// BPF opcode: `or64 dst, imm` /// `dst |= imm`. +pub const OR64_IMM: u8 = BPF_ALU64 | BPF_K | BPF_OR; +/// BPF opcode: `or64 dst, src` /// `dst |= src`. +pub const OR64_REG: u8 = BPF_ALU64 | BPF_X | BPF_OR; +/// BPF opcode: `and64 dst, imm` /// `dst &= imm`. +pub const AND64_IMM: u8 = BPF_ALU64 | BPF_K | BPF_AND; +/// BPF opcode: `and64 dst, src` /// `dst &= src`. +pub const AND64_REG: u8 = BPF_ALU64 | BPF_X | BPF_AND; +/// BPF opcode: `lsh64 dst, imm` /// `dst <<= imm`. +pub const LSH64_IMM: u8 = BPF_ALU64 | BPF_K | BPF_LSH; +/// BPF opcode: `lsh64 dst, src` /// `dst <<= src`. +pub const LSH64_REG: u8 = BPF_ALU64 | BPF_X | BPF_LSH; +/// BPF opcode: `rsh64 dst, imm` /// `dst >>= imm`. +pub const RSH64_IMM: u8 = BPF_ALU64 | BPF_K | BPF_RSH; +/// BPF opcode: `rsh64 dst, src` /// `dst >>= src`. +pub const RSH64_REG: u8 = BPF_ALU64 | BPF_X | BPF_RSH; +/// BPF opcode: `neg64 dst, imm` /// `dst = -dst`. +pub const NEG64: u8 = BPF_ALU64 | BPF_NEG; +/// BPF opcode: `mod64 dst, imm` /// `dst %= imm`. +pub const MOD64_IMM: u8 = BPF_ALU64 | BPF_K | BPF_MOD; +/// BPF opcode: `mod64 dst, src` /// `dst %= src`. +pub const MOD64_REG: u8 = BPF_ALU64 | BPF_X | BPF_MOD; +/// BPF opcode: `xor64 dst, imm` /// `dst ^= imm`. +pub const XOR64_IMM: u8 = BPF_ALU64 | BPF_K | BPF_XOR; +/// BPF opcode: `xor64 dst, src` /// `dst ^= src`. +pub const XOR64_REG: u8 = BPF_ALU64 | BPF_X | BPF_XOR; +/// BPF opcode: `mov64 dst, imm` /// `dst = imm`. +pub const MOV64_IMM: u8 = BPF_ALU64 | BPF_K | BPF_MOV; +/// BPF opcode: `mov64 dst, src` /// `dst = src`. +pub const MOV64_REG: u8 = BPF_ALU64 | BPF_X | BPF_MOV; +/// BPF opcode: `arsh64 dst, imm` /// `dst >>= imm (arithmetic)`. +/// +/// +pub const ARSH64_IMM: u8 = BPF_ALU64 | BPF_K | BPF_ARSH; +/// BPF opcode: `arsh64 dst, src` /// `dst >>= src (arithmetic)`. +/// +/// +pub const ARSH64_REG: u8 = BPF_ALU64 | BPF_X | BPF_ARSH; + +/// BPF opcode: `ja +off` /// `PC += off`. +pub const JA: u8 = BPF_JMP | BPF_JA; +/// BPF opcode: `jeq dst, imm, +off` /// `PC += off if dst == imm`. +pub const JEQ_IMM: u8 = BPF_JMP | BPF_K | BPF_JEQ; +/// BPF opcode: `jeq dst, src, +off` /// `PC += off if dst == src`. +pub const JEQ_REG: u8 = BPF_JMP | BPF_X | BPF_JEQ; +/// BPF opcode: `jgt dst, imm, +off` /// `PC += off if dst > imm`. +pub const JGT_IMM: u8 = BPF_JMP | BPF_K | BPF_JGT; +/// BPF opcode: `jgt dst, src, +off` /// `PC += off if dst > src`. +pub const JGT_REG: u8 = BPF_JMP | BPF_X | BPF_JGT; +/// BPF opcode: `jge dst, imm, +off` /// `PC += off if dst >= imm`. +pub const JGE_IMM: u8 = BPF_JMP | BPF_K | BPF_JGE; +/// BPF opcode: `jge dst, src, +off` /// `PC += off if dst >= src`. +pub const JGE_REG: u8 = BPF_JMP | BPF_X | BPF_JGE; +/// BPF opcode: `jlt dst, imm, +off` /// `PC += off if dst < imm`. +pub const JLT_IMM: u8 = BPF_JMP | BPF_K | BPF_JLT; +/// BPF opcode: `jlt dst, src, +off` /// `PC += off if dst < src`. +pub const JLT_REG: u8 = BPF_JMP | BPF_X | BPF_JLT; +/// BPF opcode: `jle dst, imm, +off` /// `PC += off if dst <= imm`. +pub const JLE_IMM: u8 = BPF_JMP | BPF_K | BPF_JLE; +/// BPF opcode: `jle dst, src, +off` /// `PC += off if dst <= src`. +pub const JLE_REG: u8 = BPF_JMP | BPF_X | BPF_JLE; +/// BPF opcode: `jset dst, imm, +off` /// `PC += off if dst & imm`. +pub const JSET_IMM: u8 = BPF_JMP | BPF_K | BPF_JSET; +/// BPF opcode: `jset dst, src, +off` /// `PC += off if dst & src`. +pub const JSET_REG: u8 = BPF_JMP | BPF_X | BPF_JSET; +/// BPF opcode: `jne dst, imm, +off` /// `PC += off if dst != imm`. +pub const JNE_IMM: u8 = BPF_JMP | BPF_K | BPF_JNE; +/// BPF opcode: `jne dst, src, +off` /// `PC += off if dst != src`. +pub const JNE_REG: u8 = BPF_JMP | BPF_X | BPF_JNE; +/// BPF opcode: `jsgt dst, imm, +off` /// `PC += off if dst > imm (signed)`. +pub const JSGT_IMM: u8 = BPF_JMP | BPF_K | BPF_JSGT; +/// BPF opcode: `jsgt dst, src, +off` /// `PC += off if dst > src (signed)`. +pub const JSGT_REG: u8 = BPF_JMP | BPF_X | BPF_JSGT; +/// BPF opcode: `jsge dst, imm, +off` /// `PC += off if dst >= imm (signed)`. +pub const JSGE_IMM: u8 = BPF_JMP | BPF_K | BPF_JSGE; +/// BPF opcode: `jsge dst, src, +off` /// `PC += off if dst >= src (signed)`. +pub const JSGE_REG: u8 = BPF_JMP | BPF_X | BPF_JSGE; +/// BPF opcode: `jslt dst, imm, +off` /// `PC += off if dst < imm (signed)`. +pub const JSLT_IMM: u8 = BPF_JMP | BPF_K | BPF_JSLT; +/// BPF opcode: `jslt dst, src, +off` /// `PC += off if dst < src (signed)`. +pub const JSLT_REG: u8 = BPF_JMP | BPF_X | BPF_JSLT; +/// BPF opcode: `jsle dst, imm, +off` /// `PC += off if dst <= imm (signed)`. +pub const JSLE_IMM: u8 = BPF_JMP | BPF_K | BPF_JSLE; +/// BPF opcode: `jsle dst, src, +off` /// `PC += off if dst <= src (signed)`. +pub const JSLE_REG: u8 = BPF_JMP | BPF_X | BPF_JSLE; + +/// BPF opcode: `jeq dst, imm, +off` /// `PC += off if (dst as u32) == imm`. +pub const JEQ_IMM32: u8 = BPF_JMP32 | BPF_K | BPF_JEQ; +/// BPF opcode: `jeq dst, src, +off` /// `PC += off if (dst as u32) == (src as u32)`. +pub const JEQ_REG32: u8 = BPF_JMP32 | BPF_X | BPF_JEQ; +/// BPF opcode: `jgt dst, imm, +off` /// `PC += off if (dst as u32) > imm`. +pub const JGT_IMM32: u8 = BPF_JMP32 | BPF_K | BPF_JGT; +/// BPF opcode: `jgt dst, src, +off` /// `PC += off if (dst as u32) > (src as u32)`. +pub const JGT_REG32: u8 = BPF_JMP32 | BPF_X | BPF_JGT; +/// BPF opcode: `jge dst, imm, +off` /// `PC += off if (dst as u32) >= imm`. +pub const JGE_IMM32: u8 = BPF_JMP32 | BPF_K | BPF_JGE; +/// BPF opcode: `jge dst, src, +off` /// `PC += off if (dst as u32) >= (src as u32)`. +pub const JGE_REG32: u8 = BPF_JMP32 | BPF_X | BPF_JGE; +/// BPF opcode: `jlt dst, imm, +off` /// `PC += off if (dst as u32) < imm`. +pub const JLT_IMM32: u8 = BPF_JMP32 | BPF_K | BPF_JLT; +/// BPF opcode: `jlt dst, src, +off` /// `PC += off if (dst as u32) < (src as u32)`. +pub const JLT_REG32: u8 = BPF_JMP32 | BPF_X | BPF_JLT; +/// BPF opcode: `jle dst, imm, +off` /// `PC += off if (dst as u32) <= imm`. +pub const JLE_IMM32: u8 = BPF_JMP32 | BPF_K | BPF_JLE; +/// BPF opcode: `jle dst, src, +off` /// `PC += off if (dst as u32) <= (src as u32)`. +pub const JLE_REG32: u8 = BPF_JMP32 | BPF_X | BPF_JLE; +/// BPF opcode: `jset dst, imm, +off` /// `PC += off if (dst as u32) & imm`. +pub const JSET_IMM32: u8 = BPF_JMP32 | BPF_K | BPF_JSET; +/// BPF opcode: `jset dst, src, +off` /// `PC += off if (dst as u32) & (src as u32)`. +pub const JSET_REG32: u8 = BPF_JMP32 | BPF_X | BPF_JSET; +/// BPF opcode: `jne dst, imm, +off` /// `PC += off if (dst as u32) != imm`. +pub const JNE_IMM32: u8 = BPF_JMP32 | BPF_K | BPF_JNE; +/// BPF opcode: `jne dst, src, +off` /// `PC += off if (dst as u32) != (src as u32)`. +pub const JNE_REG32: u8 = BPF_JMP32 | BPF_X | BPF_JNE; +/// BPF opcode: `jsgt dst, imm, +off` /// `PC += off if (dst as i32) > imm (signed)`. +pub const JSGT_IMM32: u8 = BPF_JMP32 | BPF_K | BPF_JSGT; +/// BPF opcode: `jsgt dst, src, +off` /// `PC += off if (dst as i32) > (src as i32) (signed)`. +pub const JSGT_REG32: u8 = BPF_JMP32 | BPF_X | BPF_JSGT; +/// BPF opcode: `jsge dst, imm, +off` /// `PC += off if (dst as i32) >= imm (signed)`. +pub const JSGE_IMM32: u8 = BPF_JMP32 | BPF_K | BPF_JSGE; +/// BPF opcode: `jsge dst, src, +off` /// `PC += off if (dst as i32) >= (src as i32) (signed)`. +pub const JSGE_REG32: u8 = BPF_JMP32 | BPF_X | BPF_JSGE; +/// BPF opcode: `jslt dst, imm, +off` /// `PC += off if (dst as i32) < imm (signed)`. +pub const JSLT_IMM32: u8 = BPF_JMP32 | BPF_K | BPF_JSLT; +/// BPF opcode: `jslt dst, src, +off` /// `PC += off if (dst as i32) < (src as i32) (signed)`. +pub const JSLT_REG32: u8 = BPF_JMP32 | BPF_X | BPF_JSLT; +/// BPF opcode: `jsle dst, imm, +off` /// `PC += off if (dst as i32) <= imm (signed)`. +pub const JSLE_IMM32: u8 = BPF_JMP32 | BPF_K | BPF_JSLE; +/// BPF opcode: `jsle dst, src, +off` /// `PC += off if (dst as i32) <= (src as i32) (signed)`. +pub const JSLE_REG32: u8 = BPF_JMP32 | BPF_X | BPF_JSLE; + +/// BPF opcode: `call imm` /// helper function call to helper with key `imm`. +pub const CALL: u8 = BPF_JMP | BPF_CALL; +/// BPF opcode: tail call. +pub const TAIL_CALL: u8 = BPF_JMP | BPF_X | BPF_CALL; +/// BPF opcode: `exit` /// `return r0`. +pub const EXIT: u8 = BPF_JMP | BPF_EXIT; + +// Used in JIT +/// Mask to extract the operation class from an operation code. +pub const BPF_CLS_MASK: u8 = 0x07; +/// Mask to extract the arithmetic operation code from an instruction operation code. +pub const BPF_ALU_OP_MASK: u8 = 0xf0; + +/// Prototype of an eBPF helper function. +pub type Helper = fn(u64, u64, u64, u64, u64) -> u64; + +/// An eBPF instruction. +/// +/// See for the Linux kernel +/// documentation about eBPF, or for a +/// more concise version. +#[derive(Debug, PartialEq, Eq, Clone)] +pub struct Insn { + /// Operation code. + pub opc: u8, + /// Destination register operand. + pub dst: u8, + /// Source register operand. + pub src: u8, + /// Offset operand. + pub off: i16, + /// Immediate value operand. + pub imm: i32, +} + +impl Insn { + /// Turn an `Insn` back into an array of bytes. + /// + /// # Examples + /// + /// ``` + /// use rbpf::ebpf; + /// + /// let prog: &[u8] = &[ + /// 0xb7, 0x12, 0x56, 0x34, 0xde, 0xbc, 0x9a, 0x78, + /// ]; + /// let insn = ebpf::Insn { + /// opc: 0xb7, + /// dst: 2, + /// src: 1, + /// off: 0x3456, + /// imm: 0x789abcde + /// }; + /// assert_eq!(insn.to_array(), prog); + /// ``` + pub fn to_array(&self) -> [u8; INSN_SIZE] { + [ + self.opc, + self.src.wrapping_shl(4) | self.dst, + (self.off & 0xff) as u8, + self.off.wrapping_shr(8) as u8, + (self.imm & 0xff) as u8, + (self.imm & 0xff_00).wrapping_shr(8) as u8, + (self.imm as u32 & 0xff_00_00).wrapping_shr(16) as u8, + (self.imm as u32 & 0xff_00_00_00).wrapping_shr(24) as u8, + ] + } + + /// Turn an `Insn` into an vector of bytes. + /// + /// # Examples + /// + /// ``` + /// use rbpf::ebpf; + /// + /// let prog: Vec = vec![ + /// 0xb7, 0x12, 0x56, 0x34, 0xde, 0xbc, 0x9a, 0x78, + /// ]; + /// let insn = ebpf::Insn { + /// opc: 0xb7, + /// dst: 2, + /// src: 1, + /// off: 0x3456, + /// imm: 0x789abcde + /// }; + /// assert_eq!(insn.to_vec(), prog); + /// ``` + pub fn to_vec(&self) -> Vec { + vec![ + self.opc, + self.src.wrapping_shl(4) | self.dst, + (self.off & 0xff) as u8, + self.off.wrapping_shr(8) as u8, + (self.imm & 0xff) as u8, + (self.imm & 0xff_00).wrapping_shr(8) as u8, + (self.imm as u32 & 0xff_00_00).wrapping_shr(16) as u8, + (self.imm as u32 & 0xff_00_00_00).wrapping_shr(24) as u8, + ] + } +} + +/// Get the instruction at `idx` of an eBPF program. `idx` is the index (number) of the +/// instruction (not a byte offset). The first instruction has index 0. +/// +/// # Panics +/// +/// Panics if it is not possible to get the instruction (if idx is too high, or last instruction is +/// incomplete). +/// +/// # Examples +/// +/// ``` +/// use rbpf::ebpf; +/// +/// let prog = &[ +/// 0xb7, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +/// 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 +/// ]; +/// let insn = ebpf::get_insn(prog, 1); +/// assert_eq!(insn.opc, 0x95); +/// ``` +/// +/// The example below will panic, since the last instruction is not complete and cannot be loaded. +/// +/// ```rust,should_panic +/// use rbpf::ebpf; +/// +/// let prog = &[ +/// 0xb7, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +/// 0x95, 0x00, 0x00, 0x00, 0x00, 0x00 // two bytes missing +/// ]; +/// let insn = ebpf::get_insn(prog, 1); +/// ``` +pub fn get_insn(prog: &[u8], idx: usize) -> Insn { + // This guard should not be needed in most cases, since the verifier already checks the program + // size, and indexes should be fine in the interpreter/JIT. But this function is publicly + // available and user can call it with any `idx`, so we have to check anyway. + if (idx + 1) * INSN_SIZE > prog.len() { + panic!( + "Error: cannot reach instruction at index {:?} in program containing {:?} bytes", + idx, + prog.len() + ); + } + Insn { + opc: prog[INSN_SIZE * idx], + dst: prog[INSN_SIZE * idx + 1] & 0x0f, + src: (prog[INSN_SIZE * idx + 1] & 0xf0) >> 4, + off: LittleEndian::read_i16(&prog[(INSN_SIZE * idx + 2)..]), + imm: LittleEndian::read_i32(&prog[(INSN_SIZE * idx + 4)..]), + } +} + +/// Return a vector of `struct Insn` built from a program. +/// +/// This is provided as a convenience for users wishing to manipulate a vector of instructions, for +/// example for dumping the program instruction after instruction with a custom format. +/// +/// Note that the two parts of `LD_DW_IMM` instructions (spanning on 64 bits) are considered as two +/// distinct instructions. +/// +/// # Examples +/// +/// ``` +/// use rbpf::ebpf; +/// +/// let prog = &[ +/// 0x18, 0x00, 0x00, 0x00, 0x88, 0x77, 0x66, 0x55, +/// 0x00, 0x00, 0x00, 0x00, 0x44, 0x33, 0x22, 0x11, +/// 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 +/// ]; +/// +/// let v = ebpf::to_insn_vec(prog); +/// assert_eq!(v, vec![ +/// ebpf::Insn { +/// opc: 0x18, +/// dst: 0, +/// src: 0, +/// off: 0, +/// imm: 0x55667788 +/// }, +/// ebpf::Insn { +/// opc: 0, +/// dst: 0, +/// src: 0, +/// off: 0, +/// imm: 0x11223344 +/// }, +/// ebpf::Insn { +/// opc: 0x95, +/// dst: 0, +/// src: 0, +/// off: 0, +/// imm: 0 +/// }, +/// ]); +/// ``` +pub fn to_insn_vec(prog: &[u8]) -> Vec { + if prog.len() % INSN_SIZE != 0 { + panic!( + "Error: eBPF program length must be a multiple of {:?} octets", + INSN_SIZE + ); + } + + let mut res = vec![]; + let mut insn_ptr: usize = 0; + + while insn_ptr * INSN_SIZE < prog.len() { + let insn = get_insn(prog, insn_ptr); + res.push(insn); + insn_ptr += 1; + } + res +} diff --git a/kernel/crates/rbpf/src/helpers.rs b/kernel/crates/rbpf/src/helpers.rs new file mode 100644 index 000000000..834bf8db9 --- /dev/null +++ b/kernel/crates/rbpf/src/helpers.rs @@ -0,0 +1,488 @@ +// SPDX-License-Identifier: (Apache-2.0 OR MIT) +// Copyright 2015 Big Switch Networks, Inc +// (Algorithms for uBPF helpers, originally in C) +// Copyright 2016 6WIND S.A. +// (Translation to Rust, other helpers) + +//! This module implements some built-in helpers that can be called from within an eBPF program. +//! +//! These helpers may originate from several places: +//! +//! * Some of them mimic the helpers available in the Linux kernel. +//! * Some of them were proposed as example helpers in uBPF and they were adapted here. +//! * Other helpers may be specific to rbpf. +//! +//! The prototype for helpers is always the same: five `u64` as arguments, and a `u64` as a return +//! value. Hence some helpers have unused arguments, or return a 0 value in all cases, in order to +//! respect this convention. + +// Helpers associated to kernel helpers +// See also linux/include/uapi/linux/bpf.h in Linux kernel sources. + +// bpf_ktime_getns() + +/// Index of helper `bpf_ktime_getns()`, equivalent to `bpf_time_getns()`, in Linux kernel, see +/// . +pub const BPF_KTIME_GETNS_IDX: u32 = 5; + +/// Get monotonic time (since boot time) in nanoseconds. All arguments are unused. +/// +/// # Examples +/// +/// ``` +/// use rbpf::helpers; +/// +/// let t = helpers::bpf_time_getns(0, 0, 0, 0, 0); +/// let d = t / 10u64.pow(9) / 60 / 60 / 24; +/// let h = (t / 10u64.pow(9) / 60 / 60) % 24; +/// let m = (t / 10u64.pow(9) / 60 ) % 60; +/// let s = (t / 10u64.pow(9)) % 60; +/// let ns = t % 10u64.pow(9); +/// println!("Uptime: {:#x} == {} days {}:{}:{}, {} ns", t, d, h, m, s, ns); +/// ``` +#[allow(dead_code)] +#[allow(unused_variables)] +#[allow(deprecated)] +#[cfg(feature = "std")] +pub fn bpf_time_getns(unused1: u64, unused2: u64, unused3: u64, unused4: u64, unused5: u64) -> u64 { + time::precise_time_ns() +} + +// bpf_trace_printk() + +/// Index of helper `bpf_trace_printk()`, equivalent to `bpf_trace_printf()`, in Linux kernel, see +/// . +pub const BPF_TRACE_PRINTK_IDX: u32 = 6; + +/// Prints its **last three** arguments to standard output. The **first two** arguments are +/// **unused**. Returns the number of bytes written. +/// +/// By ignoring the first two arguments, it creates a helper that will have a behavior similar to +/// the one of the equivalent helper `bpf_trace_printk()` from Linux kernel. +/// +/// # Examples +/// +/// ``` +/// use rbpf::helpers; +/// +/// let res = helpers::bpf_trace_printf(0, 0, 1, 15, 32); +/// assert_eq!(res as usize, "bpf_trace_printf: 0x1, 0xf, 0x20\n".len()); +/// ``` +/// +/// This will print `bpf_trace_printf: 0x1, 0xf, 0x20`. +/// +/// The eBPF code needed to perform the call in this example would be nearly identical to the code +/// obtained by compiling the following code from C to eBPF with clang: +/// +/// ```c +/// #include +/// #include "path/to/linux/samples/bpf/bpf_helpers.h" +/// +/// int main(struct __sk_buff *skb) +/// { +/// // Only %d %u %x %ld %lu %lx %lld %llu %llx %p %s conversion specifiers allowed. +/// // See . +/// char *fmt = "bpf_trace_printk %llx, %llx, %llx\n"; +/// return bpf_trace_printk(fmt, sizeof(fmt), 1, 15, 32); +/// } +/// ``` +/// +/// This would equally print the three numbers in `/sys/kernel/debug/tracing` file each time the +/// program is run. +#[allow(dead_code)] +#[allow(unused_variables)] +#[cfg(feature = "std")] +pub fn bpf_trace_printf(unused1: u64, unused2: u64, arg3: u64, arg4: u64, arg5: u64) -> u64 { + println!("bpf_trace_printf: {arg3:#x}, {arg4:#x}, {arg5:#x}"); + let size_arg = |x| { + if x == 0 { + 1 + } else { + (x as f64).log(16.0).floor() as u64 + 1 + } + }; + "bpf_trace_printf: 0x, 0x, 0x\n".len() as u64 + size_arg(arg3) + size_arg(arg4) + size_arg(arg5) +} + +// Helpers coming from uBPF + +/// The idea is to assemble five bytes into a single `u64`. For compatibility with the helpers API, +/// each argument must be a `u64`. +/// +/// # Examples +/// +/// ``` +/// use rbpf::helpers; +/// +/// let gathered = helpers::gather_bytes(0x11, 0x22, 0x33, 0x44, 0x55); +/// assert_eq!(gathered, 0x1122334455); +/// ``` +pub fn gather_bytes(arg1: u64, arg2: u64, arg3: u64, arg4: u64, arg5: u64) -> u64 { + arg1.wrapping_shl(32) + | arg2.wrapping_shl(24) + | arg3.wrapping_shl(16) + | arg4.wrapping_shl(8) + | arg5 +} + +/// Same as `void *memfrob(void *s, size_t n);` in `string.h` in C. See the GNU manual page (in +/// section 3) for `memfrob`. The memory is directly modified, and the helper returns 0 in all +/// cases. Arguments 3 to 5 are unused. +/// +/// # Examples +/// +/// ``` +/// use rbpf::helpers; +/// +/// let val: u64 = 0x112233; +/// let val_ptr = &val as *const u64; +/// +/// helpers::memfrob(val_ptr as u64, 8, 0, 0, 0); +/// assert_eq!(val, 0x2a2a2a2a2a3b0819); +/// helpers::memfrob(val_ptr as u64, 8, 0, 0, 0); +/// assert_eq!(val, 0x112233); +/// ``` +#[allow(unused_variables)] +pub fn memfrob(ptr: u64, len: u64, unused3: u64, unused4: u64, unused5: u64) -> u64 { + for i in 0..len { + unsafe { + let mut p = (ptr + i) as *mut u8; + *p ^= 0b101010; + } + } + 0 +} + +// TODO: Try again when asm!() is available in stable Rust. +// #![feature(asm)] +// #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] +// #[allow(unused_variables)] +// pub fn memfrob (ptr: u64, len: u64, arg3: u64, arg4: u64, arg5: u64) -> u64 { +// unsafe { +// asm!( +// "mov $0xf0, %rax" +// ::: "mov $0xf1, %rcx" +// ::: "mov $0xf2, %rdx" +// ::: "mov $0xf3, %rsi" +// ::: "mov $0xf4, %rdi" +// ::: "mov $0xf5, %r8" +// ::: "mov $0xf6, %r9" +// ::: "mov $0xf7, %r10" +// ::: "mov $0xf8, %r11" +// ); +// } +// 0 +// } + +/// Compute and return the square root of argument 1, cast as a float. Arguments 2 to 5 are +/// unused. +/// +/// # Examples +/// +/// ``` +/// use rbpf::helpers; +/// +/// let x = helpers::sqrti(9, 0, 0, 0, 0); +/// assert_eq!(x, 3); +/// ``` +#[allow(dead_code)] +#[allow(unused_variables)] +#[cfg(feature = "std")] // sqrt is only available when using `std` +pub fn sqrti(arg1: u64, unused2: u64, unused3: u64, unused4: u64, unused5: u64) -> u64 { + (arg1 as f64).sqrt() as u64 +} + +/// C-like `strcmp`, return 0 if the strings are equal, and a non-null value otherwise. +/// +/// # Examples +/// +/// ``` +/// use rbpf::helpers; +/// +/// let foo = "This is a string.\0".as_ptr() as u64; +/// let bar = "This is another sting.\0".as_ptr() as u64; +/// +/// assert!(helpers::strcmp(foo, foo, 0, 0, 0) == 0); +/// assert!(helpers::strcmp(foo, bar, 0, 0, 0) != 0); +/// ``` +#[allow(dead_code)] +#[allow(unused_variables)] +pub fn strcmp(arg1: u64, arg2: u64, arg3: u64, unused4: u64, unused5: u64) -> u64 { + // C-like strcmp, maybe shorter than converting the bytes to string and comparing? + if arg1 == 0 || arg2 == 0 { + return u64::MAX; + } + let mut a = arg1; + let mut b = arg2; + unsafe { + let mut a_val = *(a as *const u8); + let mut b_val = *(b as *const u8); + while a_val == b_val && a_val != 0 && b_val != 0 { + a += 1; + b += 1; + a_val = *(a as *const u8); + b_val = *(b as *const u8); + } + if a_val >= b_val { + (a_val - b_val) as u64 + } else { + (b_val - a_val) as u64 + } + } +} + +// Some additional helpers + +/// Returns a random u64 value comprised between `min` and `max` values (inclusive). Arguments 3 to +/// 5 are unused. +/// +/// Relies on `rand()` function from libc, so `libc::srand()` should be called once before this +/// helper is used. +/// +/// # Examples +/// +/// ``` +/// extern crate libc; +/// extern crate rbpf; +/// extern crate time; +/// +/// unsafe { +/// libc::srand(time::precise_time_ns() as u32) +/// } +/// +/// let n = rbpf::helpers::rand(3, 6, 0, 0, 0); +/// assert!(3 <= n && n <= 6); +/// ``` +#[allow(dead_code)] +#[allow(unused_variables)] +#[cfg(feature = "std")] +pub fn rand(min: u64, max: u64, unused3: u64, unused4: u64, unused5: u64) -> u64 { + let mut n = unsafe { (libc::rand() as u64).wrapping_shl(32) + libc::rand() as u64 }; + if min < max { + n = n % (max + 1 - min) + min; + }; + n +} +/// Prints the helper functions name and it's index. +#[cfg(feature = "std")] +pub fn show_helper() { + for (index, name) in BPF_FUNC_MAPPER.iter().enumerate() { + println!("{}:{}", index, name); + } +} + +/// See https://github.com/torvalds/linux/blob/master/include/uapi/linux/bpf.h +pub const BPF_FUNC_MAPPER: &[&str] = &[ + "unspec", + "map_lookup_elem", + "map_update_elem", + "map_delete_elem", + "probe_read", + "ktime_get_ns", + "trace_printk", + "get_prandom_u32", + "get_smp_processor_id", + "skb_store_bytes", + "l3_csum_replace", + "l4_csum_replace", + "tail_call", + "clone_redirect", + "get_current_pid_tgid", + "get_current_uid_gid", + "get_current_comm", + "get_cgroup_classid", + "skb_vlan_push", + "skb_vlan_pop", + "skb_get_tunnel_key", + "skb_set_tunnel_key", + "perf_event_read", + "redirect", + "get_route_realm", + "perf_event_output", + "skb_load_bytes", + "get_stackid", + "csum_diff", + "skb_get_tunnel_opt", + "skb_set_tunnel_opt", + "skb_change_proto", + "skb_change_type", + "skb_under_cgroup", + "get_hash_recalc", + "get_current_task", + "probe_write_user", + "current_task_under_cgroup", + "skb_change_tail", + "skb_pull_data", + "csum_update", + "set_hash_invalid", + "get_numa_node_id", + "skb_change_head", + "xdp_adjust_head", + "probe_read_str", + "get_socket_cookie", + "get_socket_uid", + "set_hash", + "setsockopt", + "skb_adjust_room", + "redirect_map", + "sk_redirect_map", + "sock_map_update", + "xdp_adjust_meta", + "perf_event_read_value", + "perf_prog_read_value", + "getsockopt", + "override_return", + "sock_ops_cb_flags_set", + "msg_redirect_map", + "msg_apply_bytes", + "msg_cork_bytes", + "msg_pull_data", + "bind", + "xdp_adjust_tail", + "skb_get_xfrm_state", + "get_stack", + "skb_load_bytes_relative", + "fib_lookup", + "sock_hash_update", + "msg_redirect_hash", + "sk_redirect_hash", + "lwt_push_encap", + "lwt_seg6_store_bytes", + "lwt_seg6_adjust_srh", + "lwt_seg6_action", + "rc_repeat", + "rc_keydown", + "skb_cgroup_id", + "get_current_cgroup_id", + "get_local_storage", + "sk_select_reuseport", + "skb_ancestor_cgroup_id", + "sk_lookup_tcp", + "sk_lookup_udp", + "sk_release", + "map_push_elem", + "map_pop_elem", + "map_peek_elem", + "msg_push_data", + "msg_pop_data", + "rc_pointer_rel", + "spin_lock", + "spin_unlock", + "sk_fullsock", + "tcp_sock", + "skb_ecn_set_ce", + "get_listener_sock", + "skc_lookup_tcp", + "tcp_check_syncookie", + "sysctl_get_name", + "sysctl_get_current_value", + "sysctl_get_new_value", + "sysctl_set_new_value", + "strtol", + "strtoul", + "sk_storage_get", + "sk_storage_delete", + "send_signal", + "tcp_gen_syncookie", + "skb_output", + "probe_read_user", + "probe_read_kernel", + "probe_read_user_str", + "probe_read_kernel_str", + "tcp_send_ack", + "send_signal_thread", + "jiffies64", + "read_branch_records", + "get_ns_current_pid_tgid", + "xdp_output", + "get_netns_cookie", + "get_current_ancestor_cgroup_id", + "sk_assign", + "ktime_get_boot_ns", + "seq_printf", + "seq_write", + "sk_cgroup_id", + "sk_ancestor_cgroup_id", + "ringbuf_output", + "ringbuf_reserve", + "ringbuf_submit", + "ringbuf_discard", + "ringbuf_query", + "csum_level", + "skc_to_tcp6_sock", + "skc_to_tcp_sock", + "skc_to_tcp_timewait_sock", + "skc_to_tcp_request_sock", + "skc_to_udp6_sock", + "get_task_stack", + "load_hdr_opt", + "store_hdr_opt", + "reserve_hdr_opt", + "inode_storage_get", + "inode_storage_delete", + "d_path", + "copy_from_user", + "snprintf_btf", + "seq_printf_btf", + "skb_cgroup_classid", + "redirect_neigh", + "per_cpu_ptr", + "this_cpu_ptr", + "redirect_peer", + "task_storage_get", + "task_storage_delete", + "get_current_task_btf", + "bprm_opts_set", + "ktime_get_coarse_ns", + "ima_inode_hash", + "sock_from_file", + "check_mtu", + "for_each_map_elem", + "snprintf", + "sys_bpf", + "btf_find_by_name_kind", + "sys_close", + "timer_init", + "timer_set_callback", + "timer_start", + "timer_cancel", + "get_func_ip", + "get_attach_cookie", + "task_pt_regs", + "get_branch_snapshot", + "trace_vprintk", + "skc_to_unix_sock", + "kallsyms_lookup_name", + "find_vma", + "loop", + "strncmp", + "get_func_arg", + "get_func_ret", + "get_func_arg_cnt", + "get_retval", + "set_retval", + "xdp_get_buff_len", + "xdp_load_bytes", + "xdp_store_bytes", + "copy_from_user_task", + "skb_set_tstamp", + "ima_file_hash", + "kptr_xchg", + "map_lookup_percpu_elem", + "skc_to_mptcp_sock", + "dynptr_from_mem", + "ringbuf_reserve_dynptr", + "ringbuf_submit_dynptr", + "ringbuf_discard_dynptr", + "dynptr_read", + "dynptr_write", + "dynptr_data", + "tcp_raw_gen_syncookie_ipv4", + "tcp_raw_gen_syncookie_ipv6", + "tcp_raw_check_syncookie_ipv4", + "tcp_raw_check_syncookie_ipv6", + "ktime_get_tai_ns", + "user_ringbuf_drain", + "cgrp_storage_get", + "cgrp_storage_delete", +]; diff --git a/kernel/crates/rbpf/src/insn_builder.rs b/kernel/crates/rbpf/src/insn_builder.rs new file mode 100644 index 000000000..2b8b6c0d8 --- /dev/null +++ b/kernel/crates/rbpf/src/insn_builder.rs @@ -0,0 +1,2199 @@ +// SPDX-License-Identifier: (Apache-2.0 OR MIT) +// Copyright 2017 Alex Dukhno + +//! Module provides API to create eBPF programs by Rust programming language + +use alloc::{vec, vec::Vec}; + +use crate::ebpf::*; + +/// Represents single eBPF instruction +pub trait Instruction: Sized { + /// returns instruction opt code + fn opt_code_byte(&self) -> u8; + + /// returns destination register + fn get_dst(&self) -> u8 { + self.get_insn().dst + } + + /// returns source register + fn get_src(&self) -> u8 { + self.get_insn().src + } + + /// returns offset bytes + fn get_off(&self) -> i16 { + self.get_insn().off + } + + /// returns immediate value + fn get_imm(&self) -> i32 { + self.get_insn().imm + } + + /// sets destination register + fn set_dst(mut self, dst: u8) -> Self { + self.get_insn_mut().dst = dst; + self + } + + /// sets source register + fn set_src(mut self, src: u8) -> Self { + self.get_insn_mut().src = src; + self + } + + /// sets offset bytes + fn set_off(mut self, offset: i16) -> Self { + self.get_insn_mut().off = offset; + self + } + + /// sets immediate value + fn set_imm(mut self, imm: i32) -> Self { + self.get_insn_mut().imm = imm; + self + } + + /// get `ebpf::Insn` struct + fn get_insn(&self) -> &Insn; + + /// get mutable `ebpf::Insn` struct + fn get_insn_mut(&mut self) -> &mut Insn; +} + +/// General trait for `Instruction`s and `BpfCode`. +/// Provides functionality to transform `struct` into collection of bytes +pub trait IntoBytes { + /// type of targeted transformation + type Bytes; + + /// consume `Self` with transformation into `Self::Bytes` + fn into_bytes(self) -> Self::Bytes; +} + +/// General implementation of `IntoBytes` for `Instruction` +impl<'i, I: Instruction> IntoBytes for &'i I { + type Bytes = Vec; + + /// transform immutable reference of `Instruction` into `Vec` with size of 8 + /// [ 1 byte , 1 byte , 2 bytes, 4 bytes ] + /// [ OP_CODE, SRC_REG | DST_REG, OFFSET , IMMEDIATE ] + fn into_bytes(self) -> Self::Bytes { + let buffer = vec![ + self.opt_code_byte(), + self.get_src() << 4 | self.get_dst(), + self.get_off() as u8, + (self.get_off() >> 8) as u8, + self.get_imm() as u8, + (self.get_imm() >> 8) as u8, + (self.get_imm() >> 16) as u8, + (self.get_imm() >> 24) as u8, + ]; + buffer + } +} + +/// BPF instruction stack in byte representation +#[derive(Default)] +pub struct BpfCode { + instructions: Vec, +} + +impl BpfCode { + /// creates new empty BPF instruction stack + pub fn new() -> Self { + BpfCode { + instructions: vec![], + } + } + + /// create ADD instruction + pub fn add(&mut self, source: Source, arch: Arch) -> Move { + self.mov_internal(source, arch, OpBits::Add) + } + + /// create SUB instruction + pub fn sub(&mut self, source: Source, arch: Arch) -> Move { + self.mov_internal(source, arch, OpBits::Sub) + } + + /// create MUL instruction + pub fn mul(&mut self, source: Source, arch: Arch) -> Move { + self.mov_internal(source, arch, OpBits::Mul) + } + + /// create DIV instruction + pub fn div(&mut self, source: Source, arch: Arch) -> Move { + self.mov_internal(source, arch, OpBits::Div) + } + + /// create OR instruction + pub fn bit_or(&mut self, source: Source, arch: Arch) -> Move { + self.mov_internal(source, arch, OpBits::BitOr) + } + + /// create AND instruction + pub fn bit_and(&mut self, source: Source, arch: Arch) -> Move { + self.mov_internal(source, arch, OpBits::BitAnd) + } + + /// create LSHIFT instruction + pub fn left_shift(&mut self, source: Source, arch: Arch) -> Move { + self.mov_internal(source, arch, OpBits::LShift) + } + + /// create RSHIFT instruction + pub fn right_shift(&mut self, source: Source, arch: Arch) -> Move { + self.mov_internal(source, arch, OpBits::RShift) + } + + /// create NEGATE instruction + pub fn negate(&mut self, arch: Arch) -> Move { + self.mov_internal(Source::Imm, arch, OpBits::Negate) + } + + /// create MOD instruction + pub fn modulo(&mut self, source: Source, arch: Arch) -> Move { + self.mov_internal(source, arch, OpBits::Mod) + } + + /// create XOR instruction + pub fn bit_xor(&mut self, source: Source, arch: Arch) -> Move { + self.mov_internal(source, arch, OpBits::BitXor) + } + + /// create MOV instruction + pub fn mov(&mut self, source: Source, arch: Arch) -> Move { + self.mov_internal(source, arch, OpBits::Mov) + } + + /// create SIGNED RSHIFT instruction + pub fn signed_right_shift(&mut self, source: Source, arch: Arch) -> Move { + self.mov_internal(source, arch, OpBits::SignRShift) + } + + #[inline] + fn mov_internal(&mut self, source: Source, arch_bits: Arch, op_bits: OpBits) -> Move { + Move { + bpf_code: self, + src_bit: source, + op_bits, + arch_bits, + insn: Insn { + opc: 0x00, + dst: 0x00, + src: 0x00, + off: 0x00_00, + imm: 0x00_00_00_00, + }, + } + } + + /// create byte swap instruction + pub fn swap_bytes(&mut self, endian: Endian) -> SwapBytes { + SwapBytes { + bpf_code: self, + endian, + insn: Insn { + opc: 0x00, + dst: 0x00, + src: 0x00, + off: 0x00_00, + imm: 0x00_00_00_00, + }, + } + } + + /// create LOAD instruction, IMMEDIATE is the source + pub fn load(&mut self, mem_size: MemSize) -> Load { + self.load_internal(mem_size, Addressing::Imm, BPF_LD) + } + + /// create ABSOLUTE LOAD instruction + pub fn load_abs(&mut self, mem_size: MemSize) -> Load { + self.load_internal(mem_size, Addressing::Abs, BPF_LD) + } + + /// create INDIRECT LOAD instruction + pub fn load_ind(&mut self, mem_size: MemSize) -> Load { + self.load_internal(mem_size, Addressing::Ind, BPF_LD) + } + + /// create LOAD instruction, MEMORY is the source + pub fn load_x(&mut self, mem_size: MemSize) -> Load { + self.load_internal(mem_size, Addressing::Mem, BPF_LDX) + } + + #[inline] + fn load_internal(&mut self, mem_size: MemSize, addressing: Addressing, source: u8) -> Load { + Load { + bpf_code: self, + addressing, + mem_size, + source, + insn: Insn { + opc: 0x00, + dst: 0x00, + src: 0x00, + off: 0x00_00, + imm: 0x00_00_00_00, + }, + } + } + + /// creates STORE instruction, IMMEDIATE is the source + pub fn store(&mut self, mem_size: MemSize) -> Store { + self.store_internal(mem_size, BPF_IMM) + } + + /// creates STORE instruction, MEMORY is the source + pub fn store_x(&mut self, mem_size: MemSize) -> Store { + self.store_internal(mem_size, BPF_MEM | BPF_STX) + } + + #[inline] + fn store_internal(&mut self, mem_size: MemSize, source: u8) -> Store { + Store { + bpf_code: self, + mem_size, + source, + insn: Insn { + opc: 0x00, + dst: 0x00, + src: 0x00, + off: 0x00_00, + imm: 0x00_00_00_00, + }, + } + } + + /// create unconditional JMP instruction + pub fn jump_unconditional(&mut self) -> Jump { + self.jump_conditional(Cond::Abs, Source::Imm) + } + + /// create conditional JMP instruction + pub fn jump_conditional(&mut self, cond: Cond, src_bit: Source) -> Jump { + Jump { + bpf_code: self, + cond, + src_bit, + insn: Insn { + opc: 0x00, + dst: 0x00, + src: 0x00, + off: 0x00_00, + imm: 0x00_00_00_00, + }, + } + } + + /// create CALL instruction + pub fn call(&mut self) -> FunctionCall { + FunctionCall { + bpf_code: self, + insn: Insn { + opc: 0x00, + dst: 0x00, + src: 0x00, + off: 0x00_00, + imm: 0x00_00_00_00, + }, + } + } + + /// create EXIT instruction + pub fn exit(&mut self) -> Exit { + Exit { + bpf_code: self, + insn: Insn { + opc: 0x00, + dst: 0x00, + src: 0x00, + off: 0x00_00, + imm: 0x00_00_00_00, + }, + } + } +} + +/// Transform `BpfCode` into assemble representation +impl<'a> IntoBytes for &'a BpfCode { + type Bytes = &'a [u8]; + + /// returns `BpfCode` instruction stack as `&[u8]` + fn into_bytes(self) -> Self::Bytes { + self.instructions.as_slice() + } +} + +/// struct to represent `MOV ALU` instructions +pub struct Move<'i> { + bpf_code: &'i mut BpfCode, + src_bit: Source, + op_bits: OpBits, + arch_bits: Arch, + insn: Insn, +} + +impl<'i> Move<'i> { + /// push MOV instruction into BpfCode instruction stack + pub fn push(self) -> &'i mut BpfCode { + let mut asm = self.into_bytes(); + self.bpf_code.instructions.append(&mut asm); + self.bpf_code + } +} + +impl<'i> Instruction for Move<'i> { + fn opt_code_byte(&self) -> u8 { + let op_bits = self.op_bits as u8; + let src_bit = self.src_bit as u8; + let arch_bits = self.arch_bits as u8; + op_bits | src_bit | arch_bits + } + + fn get_insn_mut(&mut self) -> &mut Insn { + &mut self.insn + } + + fn get_insn(&self) -> &Insn { + &self.insn + } +} + +#[derive(Copy, Clone, PartialEq, Eq)] +/// The source of ALU and JMP instructions +pub enum Source { + /// immediate field will be used as a source + Imm = BPF_IMM as isize, + /// src register will be used as a source + Reg = BPF_X as isize, +} + +#[derive(Copy, Clone)] +enum OpBits { + Add = BPF_ADD as isize, + Sub = BPF_SUB as isize, + Mul = BPF_MUL as isize, + Div = BPF_DIV as isize, + BitOr = BPF_OR as isize, + BitAnd = BPF_AND as isize, + LShift = BPF_LSH as isize, + RShift = BPF_RSH as isize, + Negate = BPF_NEG as isize, + Mod = BPF_MOD as isize, + BitXor = BPF_XOR as isize, + Mov = BPF_MOV as isize, + SignRShift = BPF_ARSH as isize, +} + +#[derive(Copy, Clone)] +/// Architecture of instructions +pub enum Arch { + /// 64-bit instructions + X64 = BPF_ALU64 as isize, + /// 32-bit instructions + X32 = BPF_ALU as isize, +} + +/// struct representation of byte swap operation +pub struct SwapBytes<'i> { + bpf_code: &'i mut BpfCode, + endian: Endian, + insn: Insn, +} + +impl<'i> SwapBytes<'i> { + /// push bytes swap instruction into BpfCode instruction stack + pub fn push(self) -> &'i mut BpfCode { + let mut asm = self.into_bytes(); + self.bpf_code.instructions.append(&mut asm); + self.bpf_code + } +} + +impl<'i> Instruction for SwapBytes<'i> { + fn opt_code_byte(&self) -> u8 { + self.endian as u8 + } + + fn get_insn_mut(&mut self) -> &mut Insn { + &mut self.insn + } + + fn get_insn(&self) -> &Insn { + &self.insn + } +} + +#[derive(Copy, Clone)] +/// Bytes endian +pub enum Endian { + /// Little endian + Little = LE as isize, + /// Big endian + Big = BE as isize, +} + +/// struct representation of LOAD instructions +pub struct Load<'i> { + bpf_code: &'i mut BpfCode, + addressing: Addressing, + mem_size: MemSize, + source: u8, + insn: Insn, +} + +impl<'i> Load<'i> { + /// push LOAD instruction into BpfCode instruction stack + pub fn push(self) -> &'i mut BpfCode { + let mut asm = self.into_bytes(); + self.bpf_code.instructions.append(&mut asm); + self.bpf_code + } +} + +impl<'i> Instruction for Load<'i> { + fn opt_code_byte(&self) -> u8 { + let size = self.mem_size as u8; + let addressing = self.addressing as u8; + addressing | size | self.source + } + + fn get_insn_mut(&mut self) -> &mut Insn { + &mut self.insn + } + + fn get_insn(&self) -> &Insn { + &self.insn + } +} + +/// struct representation of STORE instructions +pub struct Store<'i> { + bpf_code: &'i mut BpfCode, + mem_size: MemSize, + source: u8, + insn: Insn, +} + +impl<'i> Store<'i> { + /// push STORE instruction into BpfCode instruction stack + pub fn push(self) -> &'i mut BpfCode { + let mut asm = self.into_bytes(); + self.bpf_code.instructions.append(&mut asm); + self.bpf_code + } +} + +impl<'i> Instruction for Store<'i> { + fn opt_code_byte(&self) -> u8 { + let size = self.mem_size as u8; + BPF_MEM | BPF_ST | size | self.source + } + + fn get_insn_mut(&mut self) -> &mut Insn { + &mut self.insn + } + + fn get_insn(&self) -> &Insn { + &self.insn + } +} + +#[derive(Copy, Clone)] +/// Memory size for LOAD and STORE instructions +pub enum MemSize { + /// 8-bit size + Byte = BPF_B as isize, + /// 16-bit size + HalfWord = BPF_H as isize, + /// 32-bit size + Word = BPF_W as isize, + /// 64-bit size + DoubleWord = BPF_DW as isize, +} + +#[derive(Copy, Clone)] +enum Addressing { + Imm = BPF_IMM as isize, + Abs = BPF_ABS as isize, + Ind = BPF_IND as isize, + Mem = BPF_MEM as isize, +} + +/// struct representation of JMP instructions +pub struct Jump<'i> { + bpf_code: &'i mut BpfCode, + cond: Cond, + src_bit: Source, + insn: Insn, +} + +impl<'i> Jump<'i> { + /// push JMP instruction into BpfCode instruction stack + pub fn push(self) -> &'i mut BpfCode { + let mut asm = self.into_bytes(); + self.bpf_code.instructions.append(&mut asm); + self.bpf_code + } +} + +impl<'i> Instruction for Jump<'i> { + fn opt_code_byte(&self) -> u8 { + let cmp: u8 = self.cond as u8; + let src_bit = self.src_bit as u8; + cmp | src_bit | BPF_JMP + } + + fn get_insn_mut(&mut self) -> &mut Insn { + &mut self.insn + } + + fn get_insn(&self) -> &Insn { + &self.insn + } +} + +#[derive(Copy, Clone, PartialEq, Eq)] +/// Conditions for JMP instructions +pub enum Cond { + /// Absolute or unconditional + Abs = BPF_JA as isize, + /// Jump if `==` + Equals = BPF_JEQ as isize, + /// Jump if `>` + Greater = BPF_JGT as isize, + /// Jump if `>=` + GreaterEquals = BPF_JGE as isize, + /// Jump if `<` + Lower = BPF_JLT as isize, + /// Jump if `<=` + LowerEquals = BPF_JLE as isize, + /// Jump if `src` & `dst` + BitAnd = BPF_JSET as isize, + /// Jump if `!=` + NotEquals = BPF_JNE as isize, + /// Jump if `>` (signed) + GreaterSigned = BPF_JSGT as isize, + /// Jump if `>=` (signed) + GreaterEqualsSigned = BPF_JSGE as isize, + /// Jump if `<` (signed) + LowerSigned = BPF_JSLT as isize, + /// Jump if `<=` (signed) + LowerEqualsSigned = BPF_JSLE as isize, +} + +/// struct representation of CALL instruction +pub struct FunctionCall<'i> { + bpf_code: &'i mut BpfCode, + insn: Insn, +} + +impl<'i> FunctionCall<'i> { + /// push CALL instruction into BpfCode instruction stack + pub fn push(self) -> &'i mut BpfCode { + let mut asm = self.into_bytes(); + self.bpf_code.instructions.append(&mut asm); + self.bpf_code + } +} + +impl<'i> Instruction for FunctionCall<'i> { + fn opt_code_byte(&self) -> u8 { + BPF_CALL | BPF_JMP + } + + fn get_insn_mut(&mut self) -> &mut Insn { + &mut self.insn + } + + fn get_insn(&self) -> &Insn { + &self.insn + } +} + +/// struct representation of EXIT instruction +pub struct Exit<'i> { + bpf_code: &'i mut BpfCode, + insn: Insn, +} + +impl<'i> Exit<'i> { + /// push EXIT instruction into BpfCode instruction stack + pub fn push(self) -> &'i mut BpfCode { + let mut asm = self.into_bytes(); + self.bpf_code.instructions.append(&mut asm); + self.bpf_code + } +} + +impl<'i> Instruction for Exit<'i> { + fn opt_code_byte(&self) -> u8 { + BPF_EXIT | BPF_JMP + } + + fn get_insn_mut(&mut self) -> &mut Insn { + &mut self.insn + } + + fn get_insn(&self) -> &Insn { + &self.insn + } +} + +#[cfg(test)] +mod tests { + #[cfg(test)] + mod special { + use super::super::*; + + #[test] + fn call_immediate() { + let mut program = BpfCode::new(); + program.call().set_imm(0x11_22_33_44).push(); + + assert_eq!( + program.into_bytes(), + &[0x85, 0x00, 0x00, 0x00, 0x44, 0x33, 0x22, 0x11] + ); + } + + #[test] + fn exit_operation() { + let mut program = BpfCode::new(); + program.exit().push(); + + assert_eq!( + program.into_bytes(), + &[0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + } + + #[cfg(test)] + mod jump_instructions { + #[cfg(test)] + mod register { + use super::super::super::*; + + #[test] + fn jump_on_dst_equals_src() { + let mut program = BpfCode::new(); + program + .jump_conditional(Cond::Equals, Source::Reg) + .set_dst(0x01) + .set_src(0x02) + .push(); + + assert_eq!( + program.into_bytes(), + &[0x1d, 0x21, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn jump_on_dst_greater_than_src() { + let mut program = BpfCode::new(); + program + .jump_conditional(Cond::Greater, Source::Reg) + .set_dst(0x03) + .set_src(0x02) + .push(); + + assert_eq!( + program.into_bytes(), + &[0x2d, 0x23, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn jump_on_dst_greater_or_equals_to_src() { + let mut program = BpfCode::new(); + program + .jump_conditional(Cond::GreaterEquals, Source::Reg) + .set_dst(0x04) + .set_src(0x01) + .push(); + + assert_eq!( + program.into_bytes(), + &[0x3d, 0x14, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn jump_on_dst_lower_than_src() { + let mut program = BpfCode::new(); + program + .jump_conditional(Cond::Lower, Source::Reg) + .set_dst(0x03) + .set_src(0x02) + .push(); + + assert_eq!( + program.into_bytes(), + &[0xad, 0x23, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn jump_on_dst_lower_or_equals_to_src() { + let mut program = BpfCode::new(); + program + .jump_conditional(Cond::LowerEquals, Source::Reg) + .set_dst(0x04) + .set_src(0x01) + .push(); + + assert_eq!( + program.into_bytes(), + &[0xbd, 0x14, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn jump_on_dst_bit_and_with_src_not_equal_zero() { + let mut program = BpfCode::new(); + program + .jump_conditional(Cond::BitAnd, Source::Reg) + .set_dst(0x05) + .set_src(0x02) + .push(); + + assert_eq!( + program.into_bytes(), + &[0x4d, 0x25, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn jump_on_dst_not_equals_src() { + let mut program = BpfCode::new(); + program + .jump_conditional(Cond::NotEquals, Source::Reg) + .set_dst(0x03) + .set_src(0x05) + .push(); + + assert_eq!( + program.into_bytes(), + &[0x5d, 0x53, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn jump_on_dst_greater_than_src_signed() { + let mut program = BpfCode::new(); + program + .jump_conditional(Cond::GreaterSigned, Source::Reg) + .set_dst(0x04) + .set_src(0x01) + .push(); + + assert_eq!( + program.into_bytes(), + &[0x6d, 0x14, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn jump_on_dst_greater_or_equals_src_signed() { + let mut program = BpfCode::new(); + program + .jump_conditional(Cond::GreaterEqualsSigned, Source::Reg) + .set_dst(0x01) + .set_src(0x03) + .push(); + + assert_eq!( + program.into_bytes(), + &[0x7d, 0x31, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn jump_on_dst_lower_than_src_signed() { + let mut program = BpfCode::new(); + program + .jump_conditional(Cond::LowerSigned, Source::Reg) + .set_dst(0x04) + .set_src(0x01) + .push(); + + assert_eq!( + program.into_bytes(), + &[0xcd, 0x14, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn jump_on_dst_lower_or_equals_src_signed() { + let mut program = BpfCode::new(); + program + .jump_conditional(Cond::LowerEqualsSigned, Source::Reg) + .set_dst(0x01) + .set_src(0x03) + .push(); + + assert_eq!( + program.into_bytes(), + &[0xdd, 0x31, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + } + + #[cfg(test)] + mod immediate { + use super::super::super::*; + + #[test] + fn jump_to_label() { + let mut program = BpfCode::new(); + program.jump_unconditional().set_off(0x00_11).push(); + + assert_eq!( + program.into_bytes(), + &[0x05, 0x00, 0x11, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn jump_on_dst_equals_const() { + let mut program = BpfCode::new(); + program + .jump_conditional(Cond::Equals, Source::Imm) + .set_dst(0x01) + .set_imm(0x00_11_22_33) + .push(); + + assert_eq!( + program.into_bytes(), + &[0x15, 0x01, 0x00, 0x00, 0x33, 0x22, 0x11, 0x00] + ); + } + + #[test] + fn jump_on_dst_greater_than_const() { + let mut program = BpfCode::new(); + program + .jump_conditional(Cond::Greater, Source::Imm) + .set_dst(0x02) + .set_imm(0x00_11_00_11) + .push(); + + assert_eq!( + program.into_bytes(), + &[0x25, 0x02, 0x00, 0x00, 0x11, 0x00, 0x11, 0x00] + ); + } + + #[test] + fn jump_on_dst_greater_or_equals_to_const() { + let mut program = BpfCode::new(); + program + .jump_conditional(Cond::GreaterEquals, Source::Imm) + .set_dst(0x04) + .set_imm(0x00_22_11_00) + .push(); + + assert_eq!( + program.into_bytes(), + &[0x35, 0x04, 0x00, 0x00, 0x00, 0x11, 0x22, 0x00] + ); + } + + #[test] + fn jump_on_dst_lower_than_const() { + let mut program = BpfCode::new(); + program + .jump_conditional(Cond::Lower, Source::Imm) + .set_dst(0x02) + .set_imm(0x00_11_00_11) + .push(); + + assert_eq!( + program.into_bytes(), + &[0xa5, 0x02, 0x00, 0x00, 0x11, 0x00, 0x11, 0x00] + ); + } + + #[test] + fn jump_on_dst_lower_or_equals_to_const() { + let mut program = BpfCode::new(); + program + .jump_conditional(Cond::LowerEquals, Source::Imm) + .set_dst(0x04) + .set_imm(0x00_22_11_00) + .push(); + + assert_eq!( + program.into_bytes(), + &[0xb5, 0x04, 0x00, 0x00, 0x00, 0x11, 0x22, 0x00] + ); + } + + #[test] + fn jump_on_dst_bit_and_with_const_not_equal_zero() { + let mut program = BpfCode::new(); + program + .jump_conditional(Cond::BitAnd, Source::Imm) + .set_dst(0x05) + .push(); + + assert_eq!( + program.into_bytes(), + &[0x45, 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn jump_on_dst_not_equals_const() { + let mut program = BpfCode::new(); + program + .jump_conditional(Cond::NotEquals, Source::Imm) + .set_dst(0x03) + .push(); + + assert_eq!( + program.into_bytes(), + &[0x55, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn jump_on_dst_greater_than_const_signed() { + let mut program = BpfCode::new(); + program + .jump_conditional(Cond::GreaterSigned, Source::Imm) + .set_dst(0x04) + .push(); + + assert_eq!( + program.into_bytes(), + &[0x65, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn jump_on_dst_greater_or_equals_src_signed() { + let mut program = BpfCode::new(); + program + .jump_conditional(Cond::GreaterEqualsSigned, Source::Imm) + .set_dst(0x01) + .push(); + + assert_eq!( + program.into_bytes(), + &[0x75, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn jump_on_dst_lower_than_const_signed() { + let mut program = BpfCode::new(); + program + .jump_conditional(Cond::LowerSigned, Source::Imm) + .set_dst(0x04) + .push(); + + assert_eq!( + program.into_bytes(), + &[0xc5, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn jump_on_dst_lower_or_equals_src_signed() { + let mut program = BpfCode::new(); + program + .jump_conditional(Cond::LowerEqualsSigned, Source::Imm) + .set_dst(0x01) + .push(); + + assert_eq!( + program.into_bytes(), + &[0xd5, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + } + } + + #[cfg(test)] + mod store_instructions { + use super::super::*; + + #[test] + fn store_word_from_dst_into_immediate_address() { + let mut program = BpfCode::new(); + program + .store(MemSize::Word) + .set_dst(0x01) + .set_off(0x00_11) + .set_imm(0x11_22_33_44) + .push(); + + assert_eq!( + program.into_bytes(), + &[0x62, 0x01, 0x11, 0x00, 0x44, 0x33, 0x22, 0x11] + ); + } + + #[test] + fn store_half_word_from_dst_into_immediate_address() { + let mut program = BpfCode::new(); + program + .store(MemSize::HalfWord) + .set_dst(0x02) + .set_off(0x11_22) + .push(); + + assert_eq!( + program.into_bytes(), + &[0x6a, 0x02, 0x22, 0x11, 0x00, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn store_byte_from_dst_into_immediate_address() { + let mut program = BpfCode::new(); + program.store(MemSize::Byte).push(); + + assert_eq!( + program.into_bytes(), + &[0x72, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn store_double_word_from_dst_into_immediate_address() { + let mut program = BpfCode::new(); + program.store(MemSize::DoubleWord).push(); + + assert_eq!( + program.into_bytes(), + &[0x7a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn store_word_from_dst_into_src_address() { + let mut program = BpfCode::new(); + program + .store_x(MemSize::Word) + .set_dst(0x01) + .set_src(0x02) + .push(); + + assert_eq!( + program.into_bytes(), + &[0x63, 0x21, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn store_half_word_from_dst_into_src_address() { + let mut program = BpfCode::new(); + program.store_x(MemSize::HalfWord).push(); + + assert_eq!( + program.into_bytes(), + &[0x6b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn store_byte_from_dst_into_src_address() { + let mut program = BpfCode::new(); + program.store_x(MemSize::Byte).push(); + + assert_eq!( + program.into_bytes(), + &[0x73, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn store_double_word_from_dst_into_src_address() { + let mut program = BpfCode::new(); + program.store_x(MemSize::DoubleWord).push(); + + assert_eq!( + program.into_bytes(), + &[0x7b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + } + + #[cfg(test)] + mod load_instructions { + #[cfg(test)] + mod register { + use super::super::super::*; + + #[test] + fn load_word_from_set_src_with_offset() { + let mut program = BpfCode::new(); + program + .load_x(MemSize::Word) + .set_dst(0x01) + .set_src(0x02) + .set_off(0x00_02) + .push(); + + assert_eq!( + program.into_bytes(), + &[0x61, 0x21, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn load_half_word_from_set_src_with_offset() { + let mut program = BpfCode::new(); + program + .load_x(MemSize::HalfWord) + .set_dst(0x02) + .set_src(0x01) + .set_off(0x11_22) + .push(); + + assert_eq!( + program.into_bytes(), + &[0x69, 0x12, 0x22, 0x11, 0x00, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn load_byte_from_set_src_with_offset() { + let mut program = BpfCode::new(); + program + .load_x(MemSize::Byte) + .set_dst(0x01) + .set_src(0x04) + .set_off(0x00_11) + .push(); + + assert_eq!( + program.into_bytes(), + &[0x71, 0x41, 0x11, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn load_double_word_from_set_src_with_offset() { + let mut program = BpfCode::new(); + program + .load_x(MemSize::DoubleWord) + .set_dst(0x04) + .set_src(0x05) + .set_off(0x44_55) + .push(); + + assert_eq!( + program.into_bytes(), + &[0x79, 0x54, 0x55, 0x44, 0x00, 0x00, 0x00, 0x00] + ); + } + } + + #[cfg(test)] + mod immediate { + use super::super::super::*; + + #[test] + fn load_double_word() { + let mut program = BpfCode::new(); + program + .load(MemSize::DoubleWord) + .set_dst(0x01) + .set_imm(0x00_01_02_03) + .push(); + + assert_eq!( + program.into_bytes(), + &[0x18, 0x01, 0x00, 0x00, 0x03, 0x02, 0x01, 0x00] + ); + } + + #[test] + fn load_abs_word() { + let mut program = BpfCode::new(); + program.load_abs(MemSize::Word).push(); + + assert_eq!( + program.into_bytes(), + &[0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn load_abs_half_word() { + let mut program = BpfCode::new(); + program.load_abs(MemSize::HalfWord).set_dst(0x05).push(); + + assert_eq!( + program.into_bytes(), + &[0x28, 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn load_abs_byte() { + let mut program = BpfCode::new(); + program.load_abs(MemSize::Byte).set_dst(0x01).push(); + + assert_eq!( + program.into_bytes(), + &[0x30, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn load_abs_double_word() { + let mut program = BpfCode::new(); + program + .load_abs(MemSize::DoubleWord) + .set_dst(0x01) + .set_imm(0x01_02_03_04) + .push(); + + assert_eq!( + program.into_bytes(), + &[0x38, 0x01, 0x00, 0x00, 0x04, 0x03, 0x02, 0x01] + ); + } + + #[test] + fn load_indirect_word() { + let mut program = BpfCode::new(); + program.load_ind(MemSize::Word).push(); + + assert_eq!( + program.into_bytes(), + &[0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn load_indirect_half_word() { + let mut program = BpfCode::new(); + program.load_ind(MemSize::HalfWord).push(); + + assert_eq!( + program.into_bytes(), + &[0x48, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn load_indirect_byte() { + let mut program = BpfCode::new(); + program.load_ind(MemSize::Byte).push(); + + assert_eq!( + program.into_bytes(), + &[0x50, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn load_indirect_double_word() { + let mut program = BpfCode::new(); + program.load_ind(MemSize::DoubleWord).push(); + + assert_eq!( + program.into_bytes(), + &[0x58, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + } + } + + #[cfg(test)] + mod byte_swap_instructions { + use super::super::*; + + #[test] + fn convert_host_to_little_endian_16bits() { + let mut program = BpfCode::new(); + program + .swap_bytes(Endian::Little) + .set_dst(0x01) + .set_imm(0x00_00_00_10) + .push(); + + assert_eq!( + program.into_bytes(), + &[0xd4, 0x01, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn convert_host_to_little_endian_32bits() { + let mut program = BpfCode::new(); + program + .swap_bytes(Endian::Little) + .set_dst(0x02) + .set_imm(0x00_00_00_20) + .push(); + + assert_eq!( + program.into_bytes(), + &[0xd4, 0x02, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn convert_host_to_little_endian_64bit() { + let mut program = BpfCode::new(); + program + .swap_bytes(Endian::Little) + .set_dst(0x03) + .set_imm(0x00_00_00_40) + .push(); + + assert_eq!( + program.into_bytes(), + &[0xd4, 0x03, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn convert_host_to_big_endian_16bits() { + let mut program = BpfCode::new(); + program + .swap_bytes(Endian::Big) + .set_dst(0x01) + .set_imm(0x00_00_00_10) + .push(); + + assert_eq!( + program.into_bytes(), + &[0xdc, 0x01, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn convert_host_to_big_endian_32bits() { + let mut program = BpfCode::new(); + program + .swap_bytes(Endian::Big) + .set_dst(0x02) + .set_imm(0x00_00_00_20) + .push(); + + assert_eq!( + program.into_bytes(), + &[0xdc, 0x02, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn convert_host_to_big_endian_64bit() { + let mut program = BpfCode::new(); + program + .swap_bytes(Endian::Big) + .set_dst(0x03) + .set_imm(0x00_00_00_40) + .push(); + + assert_eq!( + program.into_bytes(), + &[0xdc, 0x03, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00] + ); + } + } + + #[cfg(test)] + mod moves_instructions { + #[cfg(test)] + mod arch_x64 { + #[cfg(test)] + mod immediate { + use super::super::super::super::*; + + #[test] + fn move_and_add_const_to_register() { + let mut program = BpfCode::new(); + program + .add(Source::Imm, Arch::X64) + .set_dst(0x02) + .set_imm(0x01_02_03_04) + .push(); + + assert_eq!( + program.into_bytes(), + &[0x07, 0x02, 0x00, 0x00, 0x04, 0x03, 0x02, 0x01] + ); + } + + #[test] + fn move_sub_const_to_register() { + let mut program = BpfCode::new(); + program + .sub(Source::Imm, Arch::X64) + .set_dst(0x04) + .set_imm(0x00_01_02_03) + .push(); + + assert_eq!( + program.into_bytes(), + &[0x17, 0x04, 0x00, 0x00, 0x03, 0x02, 0x01, 0x00] + ); + } + + #[test] + fn move_mul_const_to_register() { + let mut program = BpfCode::new(); + program + .mul(Source::Imm, Arch::X64) + .set_dst(0x05) + .set_imm(0x04_03_02_01) + .push(); + + assert_eq!( + program.into_bytes(), + &[0x27, 0x05, 0x00, 0x00, 0x01, 0x02, 0x03, 0x04] + ); + } + + #[test] + fn move_div_constant_to_register() { + let mut program = BpfCode::new(); + program + .div(Source::Imm, Arch::X64) + .set_dst(0x02) + .set_imm(0x00_ff_00_ff) + .push(); + + assert_eq!( + program.into_bytes(), + &[0x37, 0x02, 0x00, 0x00, 0xff, 0x00, 0xff, 0x00] + ); + } + + #[test] + fn move_bit_or_const_to_register() { + let mut program = BpfCode::new(); + program + .bit_or(Source::Imm, Arch::X64) + .set_dst(0x02) + .set_imm(0x00_11_00_22) + .push(); + + assert_eq!( + program.into_bytes(), + &[0x47, 0x02, 0x00, 0x00, 0x22, 0x00, 0x11, 0x00] + ); + } + + #[test] + fn move_bit_and_const_to_register() { + let mut program = BpfCode::new(); + program + .bit_and(Source::Imm, Arch::X64) + .set_dst(0x02) + .set_imm(0x11_22_33_44) + .push(); + + assert_eq!( + program.into_bytes(), + &[0x57, 0x02, 0x00, 0x00, 0x44, 0x33, 0x22, 0x11] + ); + } + + #[test] + fn move_left_shift_const_to_register() { + let mut program = BpfCode::new(); + program + .left_shift(Source::Imm, Arch::X64) + .set_dst(0x01) + .push(); + + assert_eq!( + program.into_bytes(), + &[0x67, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn move_logical_right_shift_const_to_register() { + let mut program = BpfCode::new(); + program + .right_shift(Source::Imm, Arch::X64) + .set_dst(0x01) + .push(); + + assert_eq!( + program.into_bytes(), + &[0x77, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn move_negate_register() { + let mut program = BpfCode::new(); + program.negate(Arch::X64).set_dst(0x02).push(); + + assert_eq!( + program.into_bytes(), + &[0x87, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn move_mod_const_to_register() { + let mut program = BpfCode::new(); + program.modulo(Source::Imm, Arch::X64).set_dst(0x02).push(); + + assert_eq!( + program.into_bytes(), + &[0x97, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn move_bit_xor_const_to_register() { + let mut program = BpfCode::new(); + program.bit_xor(Source::Imm, Arch::X64).set_dst(0x03).push(); + + assert_eq!( + program.into_bytes(), + &[0xa7, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn move_const_to_register() { + let mut program = BpfCode::new(); + program + .mov(Source::Imm, Arch::X64) + .set_dst(0x01) + .set_imm(0x00_00_00_FF) + .push(); + + assert_eq!( + program.into_bytes(), + &[0xb7, 0x01, 0x00, 0x00, 0xff, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn move_signed_right_shift_const_to_register() { + let mut program = BpfCode::new(); + program + .signed_right_shift(Source::Imm, Arch::X64) + .set_dst(0x05) + .push(); + + assert_eq!( + program.into_bytes(), + &[0xc7, 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + } + + #[cfg(test)] + mod register { + use super::super::super::super::*; + + #[test] + fn move_and_add_from_register() { + let mut program = BpfCode::new(); + program + .add(Source::Reg, Arch::X64) + .set_dst(0x03) + .set_src(0x02) + .push(); + + assert_eq!( + program.into_bytes(), + &[0x0f, 0x23, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn move_sub_from_register_to_register() { + let mut program = BpfCode::new(); + program + .sub(Source::Reg, Arch::X64) + .set_dst(0x03) + .set_src(0x04) + .push(); + + assert_eq!( + program.into_bytes(), + &[0x1f, 0x43, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn move_mul_from_register_to_register() { + let mut program = BpfCode::new(); + program + .mul(Source::Reg, Arch::X64) + .set_dst(0x04) + .set_src(0x03) + .push(); + + assert_eq!( + program.into_bytes(), + &[0x2f, 0x34, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn move_div_from_register_to_register() { + let mut program = BpfCode::new(); + program + .div(Source::Reg, Arch::X64) + .set_dst(0x01) + .set_src(0x00) + .push(); + + assert_eq!( + program.into_bytes(), + &[0x3f, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn move_bit_or_from_register_to_register() { + let mut program = BpfCode::new(); + program + .bit_or(Source::Reg, Arch::X64) + .set_dst(0x03) + .set_src(0x01) + .push(); + + assert_eq!( + program.into_bytes(), + &[0x4f, 0x13, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn move_bit_and_from_register_to_register() { + let mut program = BpfCode::new(); + program + .bit_and(Source::Reg, Arch::X64) + .set_dst(0x03) + .set_src(0x02) + .push(); + + assert_eq!( + program.into_bytes(), + &[0x5f, 0x23, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn move_left_shift_from_register_to_register() { + let mut program = BpfCode::new(); + program + .left_shift(Source::Reg, Arch::X64) + .set_dst(0x02) + .set_src(0x03) + .push(); + + assert_eq!( + program.into_bytes(), + &[0x6f, 0x32, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn move_logical_right_shift_from_register_to_register() { + let mut program = BpfCode::new(); + program + .right_shift(Source::Reg, Arch::X64) + .set_dst(0x02) + .set_src(0x04) + .push(); + + assert_eq!( + program.into_bytes(), + &[0x7f, 0x42, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn move_mod_from_register_to_register() { + let mut program = BpfCode::new(); + program + .modulo(Source::Reg, Arch::X64) + .set_dst(0x01) + .set_src(0x02) + .push(); + + assert_eq!( + program.into_bytes(), + &[0x9f, 0x21, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn move_bit_xor_from_register_to_register() { + let mut program = BpfCode::new(); + program + .bit_xor(Source::Reg, Arch::X64) + .set_dst(0x02) + .set_src(0x04) + .push(); + + assert_eq!( + program.into_bytes(), + &[0xaf, 0x42, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn move_from_register_to_another_register() { + let mut program = BpfCode::new(); + program.mov(Source::Reg, Arch::X64).set_src(0x01).push(); + + assert_eq!( + program.into_bytes(), + &[0xbf, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn move_signed_right_shift_from_register_to_register() { + let mut program = BpfCode::new(); + program + .signed_right_shift(Source::Reg, Arch::X64) + .set_dst(0x02) + .set_src(0x03) + .push(); + + assert_eq!( + program.into_bytes(), + &[0xcf, 0x32, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + } + } + + #[cfg(test)] + mod arch_x32 { + #[cfg(test)] + mod immediate { + use super::super::super::super::*; + + #[test] + fn move_and_add_const_to_register() { + let mut program = BpfCode::new(); + program + .add(Source::Imm, Arch::X32) + .set_dst(0x02) + .set_imm(0x01_02_03_04) + .push(); + + assert_eq!( + program.into_bytes(), + &[0x04, 0x02, 0x00, 0x00, 0x04, 0x03, 0x02, 0x01] + ); + } + + #[test] + fn move_sub_const_to_register() { + let mut program = BpfCode::new(); + program + .sub(Source::Imm, Arch::X32) + .set_dst(0x04) + .set_imm(0x00_01_02_03) + .push(); + + assert_eq!( + program.into_bytes(), + &[0x14, 0x04, 0x00, 0x00, 0x03, 0x02, 0x01, 0x00] + ); + } + + #[test] + fn move_mul_const_to_register() { + let mut program = BpfCode::new(); + program + .mul(Source::Imm, Arch::X32) + .set_dst(0x05) + .set_imm(0x04_03_02_01) + .push(); + + assert_eq!( + program.into_bytes(), + &[0x24, 0x05, 0x00, 0x00, 0x01, 0x02, 0x03, 0x04] + ); + } + + #[test] + fn move_div_constant_to_register() { + let mut program = BpfCode::new(); + program + .div(Source::Imm, Arch::X32) + .set_dst(0x02) + .set_imm(0x00_ff_00_ff) + .push(); + + assert_eq!( + program.into_bytes(), + &[0x34, 0x02, 0x00, 0x00, 0xff, 0x00, 0xff, 0x00] + ); + } + + #[test] + fn move_bit_or_const_to_register() { + let mut program = BpfCode::new(); + program + .bit_or(Source::Imm, Arch::X32) + .set_dst(0x02) + .set_imm(0x00_11_00_22) + .push(); + + assert_eq!( + program.into_bytes(), + &[0x44, 0x02, 0x00, 0x00, 0x22, 0x00, 0x11, 0x00] + ); + } + + #[test] + fn move_bit_and_const_to_register() { + let mut program = BpfCode::new(); + program + .bit_and(Source::Imm, Arch::X32) + .set_dst(0x02) + .set_imm(0x11_22_33_44) + .push(); + + assert_eq!( + program.into_bytes(), + &[0x54, 0x02, 0x00, 0x00, 0x44, 0x33, 0x22, 0x11] + ); + } + + #[test] + fn move_left_shift_const_to_register() { + let mut program = BpfCode::new(); + program + .left_shift(Source::Imm, Arch::X32) + .set_dst(0x01) + .push(); + + assert_eq!( + program.into_bytes(), + &[0x64, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn move_logical_right_shift_const_to_register() { + let mut program = BpfCode::new(); + program + .right_shift(Source::Imm, Arch::X32) + .set_dst(0x01) + .push(); + + assert_eq!( + program.into_bytes(), + &[0x74, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn move_negate_register() { + let mut program = BpfCode::new(); + program.negate(Arch::X32).set_dst(0x02).push(); + + assert_eq!( + program.into_bytes(), + &[0x84, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn move_mod_const_to_register() { + let mut program = BpfCode::new(); + program.modulo(Source::Imm, Arch::X32).set_dst(0x02).push(); + + assert_eq!( + program.into_bytes(), + &[0x94, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn move_bit_xor_const_to_register() { + let mut program = BpfCode::new(); + program.bit_xor(Source::Imm, Arch::X32).set_dst(0x03).push(); + + assert_eq!( + program.into_bytes(), + &[0xa4, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn move_const_to_register() { + let mut program = BpfCode::new(); + program + .mov(Source::Imm, Arch::X32) + .set_dst(0x01) + .set_imm(0x00_00_00_FF) + .push(); + + assert_eq!( + program.into_bytes(), + &[0xb4, 0x01, 0x00, 0x00, 0xff, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn move_signed_right_shift_const_to_register() { + let mut program = BpfCode::new(); + program + .signed_right_shift(Source::Imm, Arch::X32) + .set_dst(0x05) + .push(); + + assert_eq!( + program.into_bytes(), + &[0xc4, 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + } + + #[cfg(test)] + mod register { + use super::super::super::super::*; + + #[test] + fn move_and_add_from_register() { + let mut program = BpfCode::new(); + program + .add(Source::Reg, Arch::X32) + .set_dst(0x03) + .set_src(0x02) + .push(); + + assert_eq!( + program.into_bytes(), + &[0x0c, 0x23, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn move_sub_from_register_to_register() { + let mut program = BpfCode::new(); + program + .sub(Source::Reg, Arch::X32) + .set_dst(0x03) + .set_src(0x04) + .push(); + + assert_eq!( + program.into_bytes(), + &[0x1c, 0x43, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn move_mul_from_register_to_register() { + let mut program = BpfCode::new(); + program + .mul(Source::Reg, Arch::X32) + .set_dst(0x04) + .set_src(0x03) + .push(); + + assert_eq!( + program.into_bytes(), + &[0x2c, 0x34, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn move_div_from_register_to_register() { + let mut program = BpfCode::new(); + program + .div(Source::Reg, Arch::X32) + .set_dst(0x01) + .set_src(0x00) + .push(); + + assert_eq!( + program.into_bytes(), + &[0x3c, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn move_bit_or_from_register_to_register() { + let mut program = BpfCode::new(); + program + .bit_or(Source::Reg, Arch::X32) + .set_dst(0x03) + .set_src(0x01) + .push(); + + assert_eq!( + program.into_bytes(), + &[0x4c, 0x13, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn move_bit_and_from_register_to_register() { + let mut program = BpfCode::new(); + program + .bit_and(Source::Reg, Arch::X32) + .set_dst(0x03) + .set_src(0x02) + .push(); + + assert_eq!( + program.into_bytes(), + &[0x5c, 0x23, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn move_left_shift_from_register_to_register() { + let mut program = BpfCode::new(); + program + .left_shift(Source::Reg, Arch::X32) + .set_dst(0x02) + .set_src(0x03) + .push(); + + assert_eq!( + program.into_bytes(), + &[0x6c, 0x32, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn move_logical_right_shift_from_register_to_register() { + let mut program = BpfCode::new(); + program + .right_shift(Source::Reg, Arch::X32) + .set_dst(0x02) + .set_src(0x04) + .push(); + + assert_eq!( + program.into_bytes(), + &[0x7c, 0x42, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn move_mod_from_register_to_register() { + let mut program = BpfCode::new(); + program + .modulo(Source::Reg, Arch::X32) + .set_dst(0x01) + .set_src(0x02) + .push(); + + assert_eq!( + program.into_bytes(), + &[0x9c, 0x21, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn move_bit_xor_from_register_to_register() { + let mut program = BpfCode::new(); + program + .bit_xor(Source::Reg, Arch::X32) + .set_dst(0x02) + .set_src(0x04) + .push(); + + assert_eq!( + program.into_bytes(), + &[0xac, 0x42, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn move_from_register_to_another_register() { + let mut program = BpfCode::new(); + program + .mov(Source::Reg, Arch::X32) + .set_dst(0x00) + .set_src(0x01) + .push(); + + assert_eq!( + program.into_bytes(), + &[0xbc, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn move_signed_right_shift_from_register_to_register() { + let mut program = BpfCode::new(); + program + .signed_right_shift(Source::Reg, Arch::X32) + .set_dst(0x02) + .set_src(0x03) + .push(); + + assert_eq!( + program.into_bytes(), + &[0xcc, 0x32, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + } + } + } + + #[cfg(test)] + mod programs { + use super::super::*; + + #[test] + fn example_from_assembler() { + let mut program = BpfCode::new(); + program + .add(Source::Imm, Arch::X64) + .set_dst(1) + .set_imm(0x605) + .push() + .mov(Source::Imm, Arch::X64) + .set_dst(2) + .set_imm(0x32) + .push() + .mov(Source::Reg, Arch::X64) + .set_src(0) + .set_dst(1) + .push() + .swap_bytes(Endian::Big) + .set_dst(0) + .set_imm(0x10) + .push() + .negate(Arch::X64) + .set_dst(2) + .push() + .exit() + .push(); + + let bytecode = program.into_bytes(); + let ref_prog = &[ + 0x07, 0x01, 0x00, 0x00, 0x05, 0x06, 0x00, 0x00, 0xb7, 0x02, 0x00, 0x00, 0x32, 0x00, + 0x00, 0x00, 0xbf, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xdc, 0x00, 0x00, 0x00, + 0x10, 0x00, 0x00, 0x00, 0x87, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x95, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + ]; + // cargo says: "`[{integer}; 48]` cannot be formatted using `{:?}` + // because it doesn't implement `std::fmt::Debug`" + // So let's check in two steps. + assert_eq!(bytecode[..32], ref_prog[..32]); + assert_eq!(bytecode[33..], ref_prog[33..]); + } + } +} diff --git a/kernel/crates/rbpf/src/interpreter.rs b/kernel/crates/rbpf/src/interpreter.rs new file mode 100644 index 000000000..cb4bddf3c --- /dev/null +++ b/kernel/crates/rbpf/src/interpreter.rs @@ -0,0 +1,708 @@ +// SPDX-License-Identifier: (Apache-2.0 OR MIT) +// Derived from uBPF +// Copyright 2015 Big Switch Networks, Inc +// (uBPF: VM architecture, parts of the interpreter, originally in C) +// Copyright 2016 6WIND S.A. +// (Translation to Rust, MetaBuff/multiple classes addition, hashmaps for helpers) + +use crate::{ + ebpf::{self, Insn}, + helpers::BPF_FUNC_MAPPER, + stack::StackFrame, + *, +}; + +#[cfg(not(feature = "user"))] +#[allow(unused)] +fn check_mem( + addr: u64, + len: usize, + access_type: &str, + insn_ptr: usize, + mbuff: &[u8], + mem: &[u8], + stack: &[u8], +) -> Result<(), Error> { + log::trace!( + "check_mem: addr {:#x}, len {}, access_type {}, insn_ptr {}", + addr, + len, + access_type, + insn_ptr + ); + log::trace!( + "check_mem: mbuff: {:#x}/{:#x}, mem: {:#x}/{:#x}, stack: {:#x}/{:#x}", + mbuff.as_ptr() as u64, + mbuff.len(), + mem.as_ptr() as u64, + mem.len(), + stack.as_ptr() as u64, + stack.len() + ); + Ok(()) +} + +#[cfg(feature = "user")] +fn check_mem( + addr: u64, + len: usize, + access_type: &str, + insn_ptr: usize, + mbuff: &[u8], + mem: &[u8], + stack: &[u8], +) -> Result<(), Error> { + if let Some(addr_end) = addr.checked_add(len as u64) { + if mbuff.as_ptr() as u64 <= addr && addr_end <= mbuff.as_ptr() as u64 + mbuff.len() as u64 { + return Ok(()); + } + if mem.as_ptr() as u64 <= addr && addr_end <= mem.as_ptr() as u64 + mem.len() as u64 { + return Ok(()); + } + if stack.as_ptr() as u64 <= addr && addr_end <= stack.as_ptr() as u64 + stack.len() as u64 { + return Ok(()); + } + } + + Err(Error::new(ErrorKind::Other, format!( + "Error: out of bounds memory {} (insn #{:?}), addr {:#x}, size {:?}\nmbuff: {:#x}/{:#x}, mem: {:#x}/{:#x}, stack: {:#x}/{:#x}", + access_type, insn_ptr, addr, len, + mbuff.as_ptr() as u64, mbuff.len(), + mem.as_ptr() as u64, mem.len(), + stack.as_ptr() as u64, stack.len() + ))) +} + +#[inline] +fn do_jump(insn_ptr: &mut usize, insn: &Insn) { + *insn_ptr = (*insn_ptr as i16 + insn.off) as usize; +} + +#[allow(unknown_lints)] +#[allow(cyclomatic_complexity)] +pub fn execute_program( + prog_: Option<&[u8]>, + mem: &[u8], + mbuff: &[u8], + helpers: &HashMap, +) -> Result { + const U32MAX: u64 = u32::MAX as u64; + const SHIFT_MASK_64: u64 = 0x3f; + + let prog = match prog_ { + Some(prog) => prog, + None => Err(Error::new( + ErrorKind::Other, + "Error: No program set, call prog_set() to load one", + ))?, + }; + let mut stacks = Vec::new(); + let stack = StackFrame::new(); + // R1 points to beginning of memory area, R10 to stack + let mut reg: [u64; 11] = [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + stack.as_ptr() as u64 + stack.len() as u64, + ]; + stacks.push(stack); + if !mbuff.is_empty() { + reg[1] = mbuff.as_ptr() as u64; + } else if !mem.is_empty() { + reg[1] = mem.as_ptr() as u64; + } + let check_mem_load = + |stack: &[u8], addr: u64, len: usize, insn_ptr: usize| -> Result<(), Error> { + check_mem(addr, len, "load", insn_ptr, mbuff, mem, stack) + }; + let check_mem_store = + |stack: &[u8], addr: u64, len: usize, insn_ptr: usize| -> Result<(), Error> { + check_mem(addr, len, "store", insn_ptr, mbuff, mem, stack) + }; + + // Loop on instructions + let mut insn_ptr: usize = 0; + while insn_ptr * ebpf::INSN_SIZE < prog.len() { + let insn = ebpf::get_insn(prog, insn_ptr); + insn_ptr += 1; + let _dst = insn.dst as usize; + let _src = insn.src as usize; + + match insn.opc { + // BPF_LD class + // LD_ABS_* and LD_IND_* are supposed to load pointer to data from metadata buffer. + // Since this pointer is constant, and since we already know it (mem), do not + // bother re-fetching it, just use mem already. + ebpf::LD_ABS_B => { + reg[0] = unsafe { + let x = (mem.as_ptr() as u64 + (insn.imm as u32) as u64) as *const u8; + check_mem_load(stacks.last().unwrap().as_slice(), x as u64, 8, insn_ptr)?; + x.read_unaligned() as u64 + } + } + ebpf::LD_ABS_H => { + reg[0] = unsafe { + let x = (mem.as_ptr() as u64 + (insn.imm as u32) as u64) as *const u16; + check_mem_load(stacks.last().unwrap().as_slice(), x as u64, 8, insn_ptr)?; + x.read_unaligned() as u64 + } + } + ebpf::LD_ABS_W => { + reg[0] = unsafe { + let x = (mem.as_ptr() as u64 + (insn.imm as u32) as u64) as *const u32; + check_mem_load(stacks.last().unwrap().as_slice(), x as u64, 8, insn_ptr)?; + x.read_unaligned() as u64 + } + } + ebpf::LD_ABS_DW => { + log::info!("executing LD_ABS_DW, set reg[{}] to {:#x}", _dst, insn.imm); + reg[0] = unsafe { + let x = (mem.as_ptr() as u64 + (insn.imm as u32) as u64) as *const u64; + check_mem_load(stacks.last().unwrap().as_slice(), x as u64, 8, insn_ptr)?; + x.read_unaligned() + } + } + ebpf::LD_IND_B => { + reg[0] = unsafe { + let x = + (mem.as_ptr() as u64 + reg[_src] + (insn.imm as u32) as u64) as *const u8; + check_mem_load(stacks.last().unwrap().as_slice(), x as u64, 8, insn_ptr)?; + x.read_unaligned() as u64 + } + } + ebpf::LD_IND_H => { + reg[0] = unsafe { + let x = + (mem.as_ptr() as u64 + reg[_src] + (insn.imm as u32) as u64) as *const u16; + check_mem_load(stacks.last().unwrap().as_slice(), x as u64, 8, insn_ptr)?; + x.read_unaligned() as u64 + } + } + ebpf::LD_IND_W => { + reg[0] = unsafe { + let x = + (mem.as_ptr() as u64 + reg[_src] + (insn.imm as u32) as u64) as *const u32; + check_mem_load(stacks.last().unwrap().as_slice(), x as u64, 8, insn_ptr)?; + x.read_unaligned() as u64 + } + } + ebpf::LD_IND_DW => { + reg[0] = unsafe { + let x = + (mem.as_ptr() as u64 + reg[_src] + (insn.imm as u32) as u64) as *const u64; + check_mem_load(stacks.last().unwrap().as_slice(), x as u64, 8, insn_ptr)?; + x.read_unaligned() + } + } + + ebpf::LD_DW_IMM => { + let next_insn = ebpf::get_insn(prog, insn_ptr); + insn_ptr += 1; + // log::warn!( + // "executing LD_DW_IMM, set reg[{}] to {:#x}", + // _dst, + // ((insn.imm as u32) as u64) + ((next_insn.imm as u64) << 32) + // ); + reg[_dst] = ((insn.imm as u32) as u64) + ((next_insn.imm as u64) << 32); + } + + // BPF_LDX class + ebpf::LD_B_REG => { + reg[_dst] = unsafe { + #[allow(clippy::cast_ptr_alignment)] + let x = (reg[_src] as *const u8).offset(insn.off as isize); + check_mem_load(stacks.last().unwrap().as_slice(), x as u64, 1, insn_ptr)?; + x.read_unaligned() as u64 + } + } + ebpf::LD_H_REG => { + reg[_dst] = unsafe { + #[allow(clippy::cast_ptr_alignment)] + let x = (reg[_src] as *const u8).offset(insn.off as isize) as *const u16; + check_mem_load(stacks.last().unwrap().as_slice(), x as u64, 2, insn_ptr)?; + x.read_unaligned() as u64 + } + } + ebpf::LD_W_REG => { + reg[_dst] = unsafe { + #[allow(clippy::cast_ptr_alignment)] + let x = (reg[_src] as *const u8).offset(insn.off as isize) as *const u32; + check_mem_load(stacks.last().unwrap().as_slice(), x as u64, 4, insn_ptr)?; + // log::warn!( + // "executing LD_W_REG, the ptr is REG:{} -> [{:#x}] + {:#x}", + // _src, + // reg[_src], + // insn.off + // ); + x.read_unaligned() as u64 + } + } + ebpf::LD_DW_REG => { + reg[_dst] = unsafe { + #[allow(clippy::cast_ptr_alignment)] + let x = (reg[_src] as *const u8).offset(insn.off as isize) as *const u64; + check_mem_load(stacks.last().unwrap().as_slice(), x as u64, 8, insn_ptr)?; + x.read_unaligned() + } + } + + // BPF_ST class + ebpf::ST_B_IMM => unsafe { + let x = (reg[_dst] as *const u8).offset(insn.off as isize) as *mut u8; + check_mem_store(stacks.last().unwrap().as_slice(), x as u64, 1, insn_ptr)?; + x.write_unaligned(insn.imm as u8); + }, + ebpf::ST_H_IMM => unsafe { + #[allow(clippy::cast_ptr_alignment)] + let x = (reg[_dst] as *const u8).offset(insn.off as isize) as *mut u16; + check_mem_store(stacks.last().unwrap().as_slice(), x as u64, 2, insn_ptr)?; + x.write_unaligned(insn.imm as u16); + }, + ebpf::ST_W_IMM => unsafe { + #[allow(clippy::cast_ptr_alignment)] + let x = (reg[_dst] as *const u8).offset(insn.off as isize) as *mut u32; + check_mem_store(stacks.last().unwrap().as_slice(), x as u64, 4, insn_ptr)?; + x.write_unaligned(insn.imm as u32); + }, + ebpf::ST_DW_IMM => unsafe { + #[allow(clippy::cast_ptr_alignment)] + let x = (reg[_dst] as *const u8).offset(insn.off as isize) as *mut u64; + check_mem_store(stacks.last().unwrap().as_slice(), x as u64, 8, insn_ptr)?; + x.write_unaligned(insn.imm as u64); + }, + + // BPF_STX class + ebpf::ST_B_REG => unsafe { + let x = (reg[_dst] as *const u8).offset(insn.off as isize) as *mut u8; + check_mem_store(stacks.last().unwrap().as_slice(), x as u64, 1, insn_ptr)?; + x.write_unaligned(reg[_src] as u8); + }, + ebpf::ST_H_REG => unsafe { + #[allow(clippy::cast_ptr_alignment)] + let x = (reg[_dst] as *const u8).offset(insn.off as isize) as *mut u16; + check_mem_store(stacks.last().unwrap().as_slice(), x as u64, 2, insn_ptr)?; + x.write_unaligned(reg[_src] as u16); + }, + ebpf::ST_W_REG => unsafe { + #[allow(clippy::cast_ptr_alignment)] + let x = (reg[_dst] as *const u8).offset(insn.off as isize) as *mut u32; + check_mem_store(stacks.last().unwrap().as_slice(), x as u64, 4, insn_ptr)?; + x.write_unaligned(reg[_src] as u32); + }, + ebpf::ST_DW_REG => unsafe { + #[allow(clippy::cast_ptr_alignment)] + let x = (reg[_dst] as *const u8).offset(insn.off as isize) as *mut u64; + check_mem_store(stacks.last().unwrap().as_slice(), x as u64, 8, insn_ptr)?; + x.write_unaligned(reg[_src]); + }, + ebpf::ST_W_XADD => unimplemented!(), + ebpf::ST_DW_XADD => unimplemented!(), + + // BPF_ALU class + // TODO Check how overflow works in kernel. Should we &= U32MAX all src register value + // before we do the operation? + // Cf ((0x11 << 32) - (0x1 << 32)) as u32 VS ((0x11 << 32) as u32 - (0x1 << 32) as u32 + ebpf::ADD32_IMM => reg[_dst] = (reg[_dst] as i32).wrapping_add(insn.imm) as u64, //((reg[_dst] & U32MAX) + insn.imm as u64) & U32MAX, + ebpf::ADD32_REG => reg[_dst] = (reg[_dst] as i32).wrapping_add(reg[_src] as i32) as u64, //((reg[_dst] & U32MAX) + (reg[_src] & U32MAX)) & U32MAX, + ebpf::SUB32_IMM => reg[_dst] = (reg[_dst] as i32).wrapping_sub(insn.imm) as u64, + ebpf::SUB32_REG => reg[_dst] = (reg[_dst] as i32).wrapping_sub(reg[_src] as i32) as u64, + ebpf::MUL32_IMM => reg[_dst] = (reg[_dst] as i32).wrapping_mul(insn.imm) as u64, + ebpf::MUL32_REG => reg[_dst] = (reg[_dst] as i32).wrapping_mul(reg[_src] as i32) as u64, + ebpf::DIV32_IMM if insn.imm as u32 == 0 => reg[_dst] = 0, + ebpf::DIV32_IMM => reg[_dst] = (reg[_dst] as u32 / insn.imm as u32) as u64, + ebpf::DIV32_REG if reg[_src] as u32 == 0 => reg[_dst] = 0, + ebpf::DIV32_REG => reg[_dst] = (reg[_dst] as u32 / reg[_src] as u32) as u64, + ebpf::OR32_IMM => reg[_dst] = (reg[_dst] as u32 | insn.imm as u32) as u64, + ebpf::OR32_REG => reg[_dst] = (reg[_dst] as u32 | reg[_src] as u32) as u64, + ebpf::AND32_IMM => reg[_dst] = (reg[_dst] as u32 & insn.imm as u32) as u64, + ebpf::AND32_REG => reg[_dst] = (reg[_dst] as u32 & reg[_src] as u32) as u64, + // As for the 64-bit version, we should mask the number of bits to shift with + // 0x1f, but .wrappping_shr() already takes care of it for us. + ebpf::LSH32_IMM => reg[_dst] = (reg[_dst] as u32).wrapping_shl(insn.imm as u32) as u64, + ebpf::LSH32_REG => reg[_dst] = (reg[_dst] as u32).wrapping_shl(reg[_src] as u32) as u64, + ebpf::RSH32_IMM => reg[_dst] = (reg[_dst] as u32).wrapping_shr(insn.imm as u32) as u64, + ebpf::RSH32_REG => reg[_dst] = (reg[_dst] as u32).wrapping_shr(reg[_src] as u32) as u64, + ebpf::NEG32 => { + reg[_dst] = (reg[_dst] as i32).wrapping_neg() as u64; + reg[_dst] &= U32MAX; + } + ebpf::MOD32_IMM if insn.imm as u32 == 0 => (), + ebpf::MOD32_IMM => reg[_dst] = (reg[_dst] as u32 % insn.imm as u32) as u64, + ebpf::MOD32_REG if reg[_src] as u32 == 0 => (), + ebpf::MOD32_REG => reg[_dst] = (reg[_dst] as u32 % reg[_src] as u32) as u64, + ebpf::XOR32_IMM => reg[_dst] = (reg[_dst] as u32 ^ insn.imm as u32) as u64, + ebpf::XOR32_REG => reg[_dst] = (reg[_dst] as u32 ^ reg[_src] as u32) as u64, + ebpf::MOV32_IMM => reg[_dst] = insn.imm as u32 as u64, + ebpf::MOV32_REG => reg[_dst] = (reg[_src] as u32) as u64, + // As for the 64-bit version, we should mask the number of bits to shift with + // 0x1f, but .wrappping_shr() already takes care of it for us. + ebpf::ARSH32_IMM => { + reg[_dst] = (reg[_dst] as i32).wrapping_shr(insn.imm as u32) as u64; + reg[_dst] &= U32MAX; + } + ebpf::ARSH32_REG => { + reg[_dst] = (reg[_dst] as i32).wrapping_shr(reg[_src] as u32) as u64; + reg[_dst] &= U32MAX; + } + ebpf::LE => { + reg[_dst] = match insn.imm { + 16 => (reg[_dst] as u16).to_le() as u64, + 32 => (reg[_dst] as u32).to_le() as u64, + 64 => reg[_dst].to_le(), + _ => unreachable!(), + }; + } + ebpf::BE => { + reg[_dst] = match insn.imm { + 16 => (reg[_dst] as u16).to_be() as u64, + 32 => (reg[_dst] as u32).to_be() as u64, + 64 => reg[_dst].to_be(), + _ => unreachable!(), + }; + } + + // BPF_ALU64 class + ebpf::ADD64_IMM => reg[_dst] = reg[_dst].wrapping_add(insn.imm as u64), + ebpf::ADD64_REG => reg[_dst] = reg[_dst].wrapping_add(reg[_src]), + ebpf::SUB64_IMM => reg[_dst] = reg[_dst].wrapping_sub(insn.imm as u64), + ebpf::SUB64_REG => reg[_dst] = reg[_dst].wrapping_sub(reg[_src]), + ebpf::MUL64_IMM => reg[_dst] = reg[_dst].wrapping_mul(insn.imm as u64), + ebpf::MUL64_REG => reg[_dst] = reg[_dst].wrapping_mul(reg[_src]), + ebpf::DIV64_IMM if insn.imm == 0 => reg[_dst] = 0, + ebpf::DIV64_IMM => reg[_dst] /= insn.imm as u64, + ebpf::DIV64_REG if reg[_src] == 0 => reg[_dst] = 0, + ebpf::DIV64_REG => reg[_dst] /= reg[_src], + ebpf::OR64_IMM => reg[_dst] |= insn.imm as u64, + ebpf::OR64_REG => reg[_dst] |= reg[_src], + ebpf::AND64_IMM => reg[_dst] &= insn.imm as u64, + ebpf::AND64_REG => reg[_dst] &= reg[_src], + ebpf::LSH64_IMM => reg[_dst] <<= insn.imm as u64 & SHIFT_MASK_64, + ebpf::LSH64_REG => reg[_dst] <<= reg[_src] & SHIFT_MASK_64, + ebpf::RSH64_IMM => reg[_dst] >>= insn.imm as u64 & SHIFT_MASK_64, + ebpf::RSH64_REG => reg[_dst] >>= reg[_src] & SHIFT_MASK_64, + ebpf::NEG64 => reg[_dst] = -(reg[_dst] as i64) as u64, + ebpf::MOD64_IMM if insn.imm == 0 => (), + ebpf::MOD64_IMM => reg[_dst] %= insn.imm as u64, + ebpf::MOD64_REG if reg[_src] == 0 => (), + ebpf::MOD64_REG => reg[_dst] %= reg[_src], + ebpf::XOR64_IMM => reg[_dst] ^= insn.imm as u64, + ebpf::XOR64_REG => reg[_dst] ^= reg[_src], + ebpf::MOV64_IMM => reg[_dst] = insn.imm as u64, + ebpf::MOV64_REG => reg[_dst] = reg[_src], + ebpf::ARSH64_IMM => { + reg[_dst] = (reg[_dst] as i64 >> (insn.imm as u64 & SHIFT_MASK_64)) as u64 + } + ebpf::ARSH64_REG => { + reg[_dst] = (reg[_dst] as i64 >> (reg[_src] as u64 & SHIFT_MASK_64)) as u64 + } + + // BPF_JMP class + // TODO: check this actually works as expected for signed / unsigned ops + ebpf::JA => do_jump(&mut insn_ptr, &insn), + ebpf::JEQ_IMM => { + if reg[_dst] == insn.imm as u64 { + do_jump(&mut insn_ptr, &insn); + } + } + ebpf::JEQ_REG => { + if reg[_dst] == reg[_src] { + do_jump(&mut insn_ptr, &insn); + } + } + ebpf::JGT_IMM => { + if reg[_dst] > insn.imm as u64 { + do_jump(&mut insn_ptr, &insn); + } + } + ebpf::JGT_REG => { + if reg[_dst] > reg[_src] { + do_jump(&mut insn_ptr, &insn); + } + } + ebpf::JGE_IMM => { + if reg[_dst] >= insn.imm as u64 { + do_jump(&mut insn_ptr, &insn); + } + } + ebpf::JGE_REG => { + if reg[_dst] >= reg[_src] { + do_jump(&mut insn_ptr, &insn); + } + } + ebpf::JLT_IMM => { + if reg[_dst] < insn.imm as u64 { + do_jump(&mut insn_ptr, &insn); + } + } + ebpf::JLT_REG => { + if reg[_dst] < reg[_src] { + do_jump(&mut insn_ptr, &insn); + } + } + ebpf::JLE_IMM => { + if reg[_dst] <= insn.imm as u64 { + do_jump(&mut insn_ptr, &insn); + } + } + ebpf::JLE_REG => { + if reg[_dst] <= reg[_src] { + do_jump(&mut insn_ptr, &insn); + } + } + ebpf::JSET_IMM => { + if reg[_dst] & insn.imm as u64 != 0 { + do_jump(&mut insn_ptr, &insn); + } + } + ebpf::JSET_REG => { + if reg[_dst] & reg[_src] != 0 { + do_jump(&mut insn_ptr, &insn); + } + } + ebpf::JNE_IMM => { + if reg[_dst] != insn.imm as u64 { + do_jump(&mut insn_ptr, &insn); + } + } + ebpf::JNE_REG => { + if reg[_dst] != reg[_src] { + do_jump(&mut insn_ptr, &insn); + } + } + ebpf::JSGT_IMM => { + if reg[_dst] as i64 > insn.imm as i64 { + do_jump(&mut insn_ptr, &insn); + } + } + ebpf::JSGT_REG => { + if reg[_dst] as i64 > reg[_src] as i64 { + do_jump(&mut insn_ptr, &insn); + } + } + ebpf::JSGE_IMM => { + if reg[_dst] as i64 >= insn.imm as i64 { + do_jump(&mut insn_ptr, &insn); + } + } + ebpf::JSGE_REG => { + if reg[_dst] as i64 >= reg[_src] as i64 { + do_jump(&mut insn_ptr, &insn); + } + } + ebpf::JSLT_IMM => { + if (reg[_dst] as i64) < insn.imm as i64 { + do_jump(&mut insn_ptr, &insn); + } + } + ebpf::JSLT_REG => { + if (reg[_dst] as i64) < reg[_src] as i64 { + do_jump(&mut insn_ptr, &insn); + } + } + ebpf::JSLE_IMM => { + if reg[_dst] as i64 <= insn.imm as i64 { + do_jump(&mut insn_ptr, &insn); + } + } + ebpf::JSLE_REG => { + if reg[_dst] as i64 <= reg[_src] as i64 { + do_jump(&mut insn_ptr, &insn); + } + } + + // BPF_JMP32 class + ebpf::JEQ_IMM32 => { + if reg[_dst] as u32 == insn.imm as u32 { + do_jump(&mut insn_ptr, &insn); + } + } + ebpf::JEQ_REG32 => { + if reg[_dst] as u32 == reg[_src] as u32 { + do_jump(&mut insn_ptr, &insn); + } + } + ebpf::JGT_IMM32 => { + if reg[_dst] as u32 > insn.imm as u32 { + do_jump(&mut insn_ptr, &insn); + } + } + ebpf::JGT_REG32 => { + if reg[_dst] as u32 > reg[_src] as u32 { + do_jump(&mut insn_ptr, &insn); + } + } + ebpf::JGE_IMM32 => { + if reg[_dst] as u32 >= insn.imm as u32 { + do_jump(&mut insn_ptr, &insn); + } + } + ebpf::JGE_REG32 => { + if reg[_dst] as u32 >= reg[_src] as u32 { + do_jump(&mut insn_ptr, &insn); + } + } + ebpf::JLT_IMM32 => { + if (reg[_dst] as u32) < insn.imm as u32 { + do_jump(&mut insn_ptr, &insn); + } + } + ebpf::JLT_REG32 => { + if (reg[_dst] as u32) < reg[_src] as u32 { + do_jump(&mut insn_ptr, &insn); + } + } + ebpf::JLE_IMM32 => { + if reg[_dst] as u32 <= insn.imm as u32 { + do_jump(&mut insn_ptr, &insn); + } + } + ebpf::JLE_REG32 => { + if reg[_dst] as u32 <= reg[_src] as u32 { + do_jump(&mut insn_ptr, &insn); + } + } + ebpf::JSET_IMM32 => { + if reg[_dst] as u32 & insn.imm as u32 != 0 { + do_jump(&mut insn_ptr, &insn); + } + } + ebpf::JSET_REG32 => { + if reg[_dst] as u32 & reg[_src] as u32 != 0 { + do_jump(&mut insn_ptr, &insn); + } + } + ebpf::JNE_IMM32 => { + if reg[_dst] as u32 != insn.imm as u32 { + do_jump(&mut insn_ptr, &insn); + } + } + ebpf::JNE_REG32 => { + if reg[_dst] as u32 != reg[_src] as u32 { + do_jump(&mut insn_ptr, &insn); + } + } + ebpf::JSGT_IMM32 => { + if reg[_dst] as i32 > insn.imm { + do_jump(&mut insn_ptr, &insn); + } + } + ebpf::JSGT_REG32 => { + if reg[_dst] as i32 > reg[_src] as i32 { + do_jump(&mut insn_ptr, &insn); + } + } + ebpf::JSGE_IMM32 => { + if reg[_dst] as i32 >= insn.imm { + do_jump(&mut insn_ptr, &insn); + } + } + ebpf::JSGE_REG32 => { + if reg[_dst] as i32 >= reg[_src] as i32 { + do_jump(&mut insn_ptr, &insn); + } + } + ebpf::JSLT_IMM32 => { + if (reg[_dst] as i32) < insn.imm { + do_jump(&mut insn_ptr, &insn); + } + } + ebpf::JSLT_REG32 => { + if (reg[_dst] as i32) < reg[_src] as i32 { + do_jump(&mut insn_ptr, &insn); + } + } + ebpf::JSLE_IMM32 => { + if reg[_dst] as i32 <= insn.imm { + do_jump(&mut insn_ptr, &insn); + } + } + ebpf::JSLE_REG32 => { + if reg[_dst] as i32 <= reg[_src] as i32 { + do_jump(&mut insn_ptr, &insn); + } + } + + // Do not delegate the check to the verifier, since registered functions can be + // changed after the program has been verified. + ebpf::CALL => { + // See https://www.kernel.org/doc/html/latest/bpf/standardization/instruction-set.html#id16 + let src_reg = _src; + let call_func_res = match src_reg { + 0 => { + // Handle call by address to external function. + if let Some(function) = helpers.get(&(insn.imm as u32)) { + reg[0] = function(reg[1], reg[2], reg[3], reg[4], reg[5]); + Ok(()) + }else { + Err(format!( + "Error: unknown helper function (id: {:#x}) [{}], (instruction #{})", + insn.imm as u32,BPF_FUNC_MAPPER[insn.imm as usize],insn_ptr + )) + } + } + 1 => { + // bpf to bpf call + // The function is in the same program, so we can just jump to the address + if stacks.len() >= ebpf::RBPF_MAX_CALL_DEPTH{ + Err(format!( + "Error: bpf to bpf call stack limit reached (instruction #{}) max depth: {}", + insn_ptr, ebpf::RBPF_MAX_CALL_DEPTH + )) + }else { + let mut pre_stack = stacks.last_mut().unwrap(); + // Save the callee saved registers + pre_stack.save_registers(®[6..=9]); + // Save the return address + pre_stack.save_return_address(insn_ptr as u16); + // save the stack pointer + pre_stack.save_sp(reg[10] as u16); + let mut stack = StackFrame::new(); + log::trace!("BPF TO BPF CALL: new pc: {} + {} = {}",insn_ptr ,insn.imm,insn_ptr + insn.imm as usize); + reg[10] = stack.as_ptr() as u64 + stack.len() as u64; + stacks.push(stack); + insn_ptr += insn.imm as usize; + Ok(()) + } + } + _ =>{ + Err(format!( + "Error: the function call type (id: {:#x}) [{}], (instruction #{}) not supported", + insn.imm as u32,BPF_FUNC_MAPPER[insn.imm as usize],insn_ptr + )) + } + }; + if let Err(e) = call_func_res { + Err(Error::new(ErrorKind::Other, e))?; + } + } + ebpf::TAIL_CALL => unimplemented!(), + ebpf::EXIT => { + if stacks.len() == 1 { + return Ok(reg[0]); + } else { + // Pop the stack + stacks.pop(); + let stack = stacks.last().unwrap(); + // Restore the callee saved registers + reg[6..=9].copy_from_slice(&stack.get_registers()); + // Restore the return address + insn_ptr = stack.get_return_address() as usize; + // Restore the stack pointer + reg[10] = stack.get_sp() as u64; + log::trace!("EXIT: new pc: {}", insn_ptr); + } + } + + _ => unreachable!(), + } + } + + unreachable!() +} diff --git a/kernel/crates/rbpf/src/jit.rs b/kernel/crates/rbpf/src/jit.rs new file mode 100644 index 000000000..df509ce01 --- /dev/null +++ b/kernel/crates/rbpf/src/jit.rs @@ -0,0 +1,1054 @@ +// SPDX-License-Identifier: (Apache-2.0 OR MIT) +// Derived from uBPF +// Copyright 2015 Big Switch Networks, Inc +// (uBPF: JIT algorithm, originally in C) +// Copyright 2016 6WIND S.A. +// (Translation to Rust, MetaBuff addition) + +use std::{ + fmt::{Error as FormatterError, Formatter}, + io::{Error, ErrorKind}, + mem, + ops::{Index, IndexMut}, +}; + +use crate::{ebpf, HashMap}; + +extern crate libc; + +type MachineCode = unsafe fn(*mut u8, usize, *mut u8, usize, usize, usize) -> u64; + +const PAGE_SIZE: usize = 4096; +// TODO: check how long the page must be to be sure to support an eBPF program of maximum possible +// length +const NUM_PAGES: usize = 1; + +// Special values for target_pc in struct Jump +const TARGET_OFFSET: isize = ebpf::PROG_MAX_INSNS as isize; +const TARGET_PC_EXIT: isize = TARGET_OFFSET + 1; + +#[derive(Copy, Clone)] +enum OperandSize { + S8 = 8, + S16 = 16, + S32 = 32, + S64 = 64, +} + +// Registers +const RAX: u8 = 0; +const RCX: u8 = 1; +const RDX: u8 = 2; +const RBX: u8 = 3; +const RSP: u8 = 4; +const RBP: u8 = 5; +const RSI: u8 = 6; +const RDI: u8 = 7; +const R8: u8 = 8; +const R9: u8 = 9; +const R10: u8 = 10; +const R11: u8 = 11; +//const R12: u8 = 12; +const R13: u8 = 13; +const R14: u8 = 14; +const R15: u8 = 15; + +const REGISTER_MAP_SIZE: usize = 11; +const REGISTER_MAP: [u8; REGISTER_MAP_SIZE] = [ + RAX, // 0 return value + RDI, // 1 arg 1 + RSI, // 2 arg 2 + RDX, // 3 arg 3 + R9, // 4 arg 4 + R8, // 5 arg 5 + RBX, // 6 callee-saved + R13, // 7 callee-saved + R14, // 8 callee-saved + R15, // 9 callee-saved + RBP, // 10 stack pointer + // R10 and R11 are used to compute store a constant pointer to mem and to compute offset for + // LD_ABS_* and LD_IND_* operations, so they are not mapped to any eBPF register. +]; + +// Return the x86 register for the given eBPF register +fn map_register(r: u8) -> u8 { + assert!(r < REGISTER_MAP_SIZE as u8); + REGISTER_MAP[(r % REGISTER_MAP_SIZE as u8) as usize] +} + +macro_rules! emit_bytes { + ( $mem:ident, $data:tt, $t:ty ) => {{ + let size = mem::size_of::<$t>() as usize; + assert!($mem.offset + size <= $mem.contents.len()); + unsafe { + let mut ptr = $mem.contents.as_ptr().add($mem.offset) as *mut $t; + ptr.write_unaligned($data); + } + $mem.offset += size; + }}; +} + +#[derive(Debug)] +struct Jump { + offset_loc: usize, + target_pc: isize, +} + +#[derive(Debug)] +struct JitCompiler { + pc_locs: Vec, + special_targets: HashMap, + jumps: Vec, +} + +impl JitCompiler { + fn new() -> JitCompiler { + JitCompiler { + pc_locs: vec![], + jumps: vec![], + special_targets: HashMap::new(), + } + } + + fn emit1(&self, mem: &mut JitMemory, data: u8) { + emit_bytes!(mem, data, u8); + } + + fn emit2(&self, mem: &mut JitMemory, data: u16) { + emit_bytes!(mem, data, u16); + } + + fn emit4(&self, mem: &mut JitMemory, data: u32) { + emit_bytes!(mem, data, u32); + } + + fn emit8(&self, mem: &mut JitMemory, data: u64) { + emit_bytes!(mem, data, u64); + } + + fn emit_modrm(&self, mem: &mut JitMemory, modrm: u8, r: u8, m: u8) { + assert_eq!((modrm | 0xc0), 0xc0); + self.emit1(mem, (modrm & 0xc0) | ((r & 0b111) << 3) | (m & 0b111)); + } + + fn emit_modrm_reg2reg(&self, mem: &mut JitMemory, r: u8, m: u8) { + self.emit_modrm(mem, 0xc0, r, m); + } + + fn emit_modrm_and_displacement(&self, mem: &mut JitMemory, r: u8, m: u8, d: i32) { + if d == 0 && (m & 0b111) != RBP { + self.emit_modrm(mem, 0x00, r, m); + } else if (-128..=127).contains(&d) { + self.emit_modrm(mem, 0x40, r, m); + self.emit1(mem, d as u8); + } else { + self.emit_modrm(mem, 0x80, r, m); + self.emit4(mem, d as u32); + } + } + + fn basix_rex_would_set_bits(&self, w: u8, src: u8, dst: u8) -> bool { + w != 0 || (src & 0b1000) != 0 || (dst & 0b1000) != 0 + } + + fn emit_rex(&self, mem: &mut JitMemory, w: u8, r: u8, x: u8, b: u8) { + assert_eq!((w | 1), 1); + assert_eq!((r | 1), 1); + assert_eq!((x | 1), 1); + assert_eq!((b | 1), 1); + self.emit1(mem, 0x40 | (w << 3) | (r << 2) | (x << 1) | b); + } + + // Emits a REX prefix with the top bit of src and dst. + // Skipped if no bits would be set. + fn emit_basic_rex(&self, mem: &mut JitMemory, w: u8, src: u8, dst: u8) { + if self.basix_rex_would_set_bits(w, src, dst) { + let is_masked = |val, mask| match val & mask { + 0 => 0, + _ => 1, + }; + self.emit_rex(mem, w, is_masked(src, 8), 0, is_masked(dst, 8)); + } + } + + fn emit_push(&self, mem: &mut JitMemory, r: u8) { + self.emit_basic_rex(mem, 0, 0, r); + self.emit1(mem, 0x50 | (r & 0b111)); + } + + fn emit_pop(&self, mem: &mut JitMemory, r: u8) { + self.emit_basic_rex(mem, 0, 0, r); + self.emit1(mem, 0x58 | (r & 0b111)); + } + + // REX prefix and ModRM byte + // We use the MR encoding when there is a choice + // 'src' is often used as an opcode extension + fn emit_alu32(&self, mem: &mut JitMemory, op: u8, src: u8, dst: u8) { + self.emit_basic_rex(mem, 0, src, dst); + self.emit1(mem, op); + self.emit_modrm_reg2reg(mem, src, dst); + } + + // REX prefix, ModRM byte, and 32-bit immediate + fn emit_alu32_imm32(&self, mem: &mut JitMemory, op: u8, src: u8, dst: u8, imm: i32) { + self.emit_alu32(mem, op, src, dst); + self.emit4(mem, imm as u32); + } + + // REX prefix, ModRM byte, and 8-bit immediate + fn emit_alu32_imm8(&self, mem: &mut JitMemory, op: u8, src: u8, dst: u8, imm: i8) { + self.emit_alu32(mem, op, src, dst); + self.emit1(mem, imm as u8); + } + + // REX.W prefix and ModRM byte + // We use the MR encoding when there is a choice + // 'src' is often used as an opcode extension + fn emit_alu64(&self, mem: &mut JitMemory, op: u8, src: u8, dst: u8) { + self.emit_basic_rex(mem, 1, src, dst); + self.emit1(mem, op); + self.emit_modrm_reg2reg(mem, src, dst); + } + + // REX.W prefix, ModRM byte, and 32-bit immediate + fn emit_alu64_imm32(&self, mem: &mut JitMemory, op: u8, src: u8, dst: u8, imm: i32) { + self.emit_alu64(mem, op, src, dst); + self.emit4(mem, imm as u32); + } + + // REX.W prefix, ModRM byte, and 8-bit immediate + fn emit_alu64_imm8(&self, mem: &mut JitMemory, op: u8, src: u8, dst: u8, imm: i8) { + self.emit_alu64(mem, op, src, dst); + self.emit1(mem, imm as u8); + } + + // Register to register mov + fn emit_mov(&self, mem: &mut JitMemory, src: u8, dst: u8) { + self.emit_alu64(mem, 0x89, src, dst); + } + + fn emit_cmp_imm32(&self, mem: &mut JitMemory, dst: u8, imm: i32) { + self.emit_alu64_imm32(mem, 0x81, 7, dst, imm); + } + + fn emit_cmp(&self, mem: &mut JitMemory, src: u8, dst: u8) { + self.emit_alu64(mem, 0x39, src, dst); + } + + fn emit_cmp32_imm32(&self, mem: &mut JitMemory, dst: u8, imm: i32) { + self.emit_alu32_imm32(mem, 0x81, 7, dst, imm); + } + + fn emit_cmp32(&self, mem: &mut JitMemory, src: u8, dst: u8) { + self.emit_alu32(mem, 0x39, src, dst); + } + + // Load [src + offset] into dst + fn emit_load(&self, mem: &mut JitMemory, size: OperandSize, src: u8, dst: u8, offset: i32) { + let data = match size { + OperandSize::S64 => 1, + _ => 0, + }; + self.emit_basic_rex(mem, data, dst, src); + + match size { + OperandSize::S8 => { + // movzx + self.emit1(mem, 0x0f); + self.emit1(mem, 0xb6); + } + OperandSize::S16 => { + // movzx + self.emit1(mem, 0x0f); + self.emit1(mem, 0xb7); + } + OperandSize::S32 | OperandSize::S64 => { + // mov + self.emit1(mem, 0x8b); + } + } + + self.emit_modrm_and_displacement(mem, dst, src, offset); + } + + // Load sign-extended immediate into register + fn emit_load_imm(&self, mem: &mut JitMemory, dst: u8, imm: i64) { + if imm >= i32::MIN as i64 && imm <= i32::MAX as i64 { + self.emit_alu64_imm32(mem, 0xc7, 0, dst, imm as i32); + } else { + // movabs $imm,dst + self.emit_basic_rex(mem, 1, 0, dst); + self.emit1(mem, 0xb8 | (dst & 0b111)); + self.emit8(mem, imm as u64); + } + } + + // Store register src to [dst + offset] + fn emit_store(&self, mem: &mut JitMemory, size: OperandSize, src: u8, dst: u8, offset: i32) { + match size { + OperandSize::S16 => self.emit1(mem, 0x66), // 16-bit override + _ => {} + }; + let (is_s8, is_u64, rexw) = match size { + OperandSize::S8 => (true, false, 0), + OperandSize::S64 => (false, true, 1), + _ => (false, false, 0), + }; + if is_u64 || (src & 0b1000) != 0 || (dst & 0b1000) != 0 || is_s8 { + let is_masked = |val, mask| match val & mask { + 0 => 0, + _ => 1, + }; + self.emit_rex(mem, rexw, is_masked(src, 8), 0, is_masked(dst, 8)); + } + match size { + OperandSize::S8 => self.emit1(mem, 0x88), + _ => self.emit1(mem, 0x89), + }; + self.emit_modrm_and_displacement(mem, src, dst, offset); + } + + // Store immediate to [dst + offset] + fn emit_store_imm32( + &self, + mem: &mut JitMemory, + size: OperandSize, + dst: u8, + offset: i32, + imm: i32, + ) { + match size { + OperandSize::S16 => self.emit1(mem, 0x66), // 16-bit override + _ => {} + }; + match size { + OperandSize::S64 => self.emit_basic_rex(mem, 1, 0, dst), + _ => self.emit_basic_rex(mem, 0, 0, dst), + }; + match size { + OperandSize::S8 => self.emit1(mem, 0xc6), + _ => self.emit1(mem, 0xc7), + }; + self.emit_modrm_and_displacement(mem, 0, dst, offset); + match size { + OperandSize::S8 => self.emit1(mem, imm as u8), + OperandSize::S16 => self.emit2(mem, imm as u16), + _ => self.emit4(mem, imm as u32), + }; + } + + fn emit_direct_jcc(&self, mem: &mut JitMemory, code: u8, offset: u32) { + self.emit1(mem, 0x0f); + self.emit1(mem, code); + emit_bytes!(mem, offset, u32); + } + + fn emit_call(&self, mem: &mut JitMemory, target: usize) { + // TODO use direct call when possible + self.emit_load_imm(mem, RAX, target as i64); + // callq *%rax + self.emit1(mem, 0xff); + self.emit1(mem, 0xd0); + } + + fn emit_jump_offset(&mut self, mem: &mut JitMemory, target_pc: isize) { + let jump = Jump { + offset_loc: mem.offset, + target_pc, + }; + self.jumps.push(jump); + self.emit4(mem, 0); + } + + fn emit_jcc(&mut self, mem: &mut JitMemory, code: u8, target_pc: isize) { + self.emit1(mem, 0x0f); + self.emit1(mem, code); + self.emit_jump_offset(mem, target_pc); + } + + fn emit_jmp(&mut self, mem: &mut JitMemory, target_pc: isize) { + self.emit1(mem, 0xe9); + self.emit_jump_offset(mem, target_pc); + } + + fn set_anchor(&mut self, mem: &mut JitMemory, target: isize) { + self.special_targets.insert(target, mem.offset); + } + + fn emit_muldivmod( + &mut self, + mem: &mut JitMemory, + pc: u16, + opc: u8, + src: u8, + dst: u8, + imm: i32, + ) { + let mul = (opc & ebpf::BPF_ALU_OP_MASK) == (ebpf::MUL32_IMM & ebpf::BPF_ALU_OP_MASK); + let div = (opc & ebpf::BPF_ALU_OP_MASK) == (ebpf::DIV32_IMM & ebpf::BPF_ALU_OP_MASK); + let modrm = (opc & ebpf::BPF_ALU_OP_MASK) == (ebpf::MOD32_IMM & ebpf::BPF_ALU_OP_MASK); + let is64 = (opc & ebpf::BPF_CLS_MASK) == ebpf::BPF_ALU64; + let is_reg = (opc & ebpf::BPF_X) == ebpf::BPF_X; + + if (div || mul) && !is_reg && imm == 0 { + // Division by zero returns 0 + // Set register to 0: xor with itself + self.emit_alu32(mem, 0x31, dst, dst); + return; + } + if modrm && !is_reg && imm == 0 { + // Modulo remainder of division by zero keeps destination register unchanged + return; + } + if (div || modrm) && is_reg { + self.emit_load_imm(mem, RCX, pc as i64); + + // test src,src + if is64 { + self.emit_alu64(mem, 0x85, src, src); + } else { + self.emit_alu32(mem, 0x85, src, src); + } + + if div { + // No division by 0: skip next instructions + // Jump offset: emit_alu32 adds 2 to 3 bytes, emit_jmp adds 5 + let offset = match self.basix_rex_would_set_bits(0, dst, dst) { + true => 3 + 5, + false => 2 + 5, + }; + self.emit_direct_jcc(mem, 0x85, offset); + // Division by 0: set dst to 0 then go to next instruction + // Set register to 0: xor with itself + self.emit_alu32(mem, 0x31, dst, dst); + self.emit_jmp(mem, (pc + 1) as isize); + } + if modrm { + // Modulo by zero: keep destination register unchanged + self.emit_jcc(mem, 0x84, (pc + 1) as isize); + } + } + + if dst != RAX { + self.emit_push(mem, RAX); + } + if dst != RDX { + self.emit_push(mem, RDX); + } + if imm != 0 { + self.emit_load_imm(mem, RCX, imm as i64); + } else { + self.emit_mov(mem, src, RCX); + } + + self.emit_mov(mem, dst, RAX); + + if div || modrm { + // Set register to 0: xor %edx,%edx + self.emit_alu32(mem, 0x31, RDX, RDX); + } + + if is64 { + self.emit_rex(mem, 1, 0, 0, 0); + } + + // mul %ecx or div %ecx + self.emit_alu32(mem, 0xf7, if mul { 4 } else { 6 }, RCX); + + if dst != RDX { + if modrm { + self.emit_mov(mem, RDX, dst); + } + self.emit_pop(mem, RDX); + } + if dst != RAX { + if div || mul { + self.emit_mov(mem, RAX, dst); + } + self.emit_pop(mem, RAX); + } + } + + fn jit_compile( + &mut self, + mem: &mut JitMemory, + prog: &[u8], + use_mbuff: bool, + update_data_ptr: bool, + helpers: &HashMap, + ) -> Result<(), Error> { + self.emit_push(mem, RBP); + self.emit_push(mem, RBX); + self.emit_push(mem, R13); + self.emit_push(mem, R14); + self.emit_push(mem, R15); + + // RDI: mbuff + // RSI: mbuff_len + // RDX: mem + // RCX: mem_len + // R8: mem_offset + // R9: mem_end_offset + + // Save mem pointer for use with LD_ABS_* and LD_IND_* instructions + self.emit_mov(mem, RDX, R10); + + match (use_mbuff, update_data_ptr) { + (false, _) => { + // We do not use any mbuff. Move mem pointer into register 1. + if map_register(1) != RDX { + self.emit_mov(mem, RDX, map_register(1)); + } + } + (true, false) => { + // We use a mbuff already pointing to mem and mem_end: move it to register 1. + if map_register(1) != RDI { + self.emit_mov(mem, RDI, map_register(1)); + } + } + (true, true) => { + // We have a fixed (simulated) mbuff: update mem and mem_end offset values in it. + // Store mem at mbuff + mem_offset. Trash R8. + self.emit_alu64(mem, 0x01, RDI, R8); // add mbuff to mem_offset in R8 + self.emit_store(mem, OperandSize::S64, RDX, R8, 0); // set mem at mbuff + mem_offset + // Store mem_end at mbuff + mem_end_offset. Trash R9. + self.emit_load(mem, OperandSize::S64, RDX, R8, 0); // load mem into R8 + self.emit_alu64(mem, 0x01, RCX, R8); // add mem_len to mem (= mem_end) + self.emit_alu64(mem, 0x01, RDI, R9); // add mbuff to mem_end_offset + self.emit_store(mem, OperandSize::S64, R8, R9, 0); // store mem_end + + // Move rdi into register 1 + if map_register(1) != RDI { + self.emit_mov(mem, RDI, map_register(1)); + } + } + } + + // Copy stack pointer to R10 + self.emit_mov(mem, RSP, map_register(10)); + + // Allocate stack space + self.emit_alu64_imm32(mem, 0x81, 5, RSP, ebpf::STACK_SIZE as i32); + + self.pc_locs = vec![0; prog.len() / ebpf::INSN_SIZE + 1]; + + let mut insn_ptr: usize = 0; + while insn_ptr * ebpf::INSN_SIZE < prog.len() { + let insn = ebpf::get_insn(prog, insn_ptr); + + self.pc_locs[insn_ptr] = mem.offset; + + let dst = map_register(insn.dst); + let src = map_register(insn.src); + let target_pc = insn_ptr as isize + insn.off as isize + 1; + + match insn.opc { + // BPF_LD class + // R10 is a constant pointer to mem. + ebpf::LD_ABS_B => self.emit_load(mem, OperandSize::S8, R10, RAX, insn.imm), + ebpf::LD_ABS_H => self.emit_load(mem, OperandSize::S16, R10, RAX, insn.imm), + ebpf::LD_ABS_W => self.emit_load(mem, OperandSize::S32, R10, RAX, insn.imm), + ebpf::LD_ABS_DW => self.emit_load(mem, OperandSize::S64, R10, RAX, insn.imm), + ebpf::LD_IND_B => { + self.emit_mov(mem, R10, R11); // load mem into R11 + self.emit_alu64(mem, 0x01, src, R11); // add src to R11 + self.emit_load(mem, OperandSize::S8, R11, RAX, insn.imm); // ld R0, mem[src+imm] + } + ebpf::LD_IND_H => { + self.emit_mov(mem, R10, R11); // load mem into R11 + self.emit_alu64(mem, 0x01, src, R11); // add src to R11 + self.emit_load(mem, OperandSize::S16, R11, RAX, insn.imm); // ld R0, mem[src+imm] + } + ebpf::LD_IND_W => { + self.emit_mov(mem, R10, R11); // load mem into R11 + self.emit_alu64(mem, 0x01, src, R11); // add src to R11 + self.emit_load(mem, OperandSize::S32, R11, RAX, insn.imm); // ld R0, mem[src+imm] + } + ebpf::LD_IND_DW => { + self.emit_mov(mem, R10, R11); // load mem into R11 + self.emit_alu64(mem, 0x01, src, R11); // add src to R11 + self.emit_load(mem, OperandSize::S64, R11, RAX, insn.imm); // ld R0, mem[src+imm] + } + + ebpf::LD_DW_IMM => { + insn_ptr += 1; + let second_part = ebpf::get_insn(prog, insn_ptr).imm as u64; + let imm = (insn.imm as u32) as u64 | second_part.wrapping_shl(32); + self.emit_load_imm(mem, dst, imm as i64); + } + + // BPF_LDX class + ebpf::LD_B_REG => self.emit_load(mem, OperandSize::S8, src, dst, insn.off as i32), + ebpf::LD_H_REG => self.emit_load(mem, OperandSize::S16, src, dst, insn.off as i32), + ebpf::LD_W_REG => self.emit_load(mem, OperandSize::S32, src, dst, insn.off as i32), + ebpf::LD_DW_REG => self.emit_load(mem, OperandSize::S64, src, dst, insn.off as i32), + + // BPF_ST class + ebpf::ST_B_IMM => { + self.emit_store_imm32(mem, OperandSize::S8, dst, insn.off as i32, insn.imm) + } + ebpf::ST_H_IMM => { + self.emit_store_imm32(mem, OperandSize::S16, dst, insn.off as i32, insn.imm) + } + ebpf::ST_W_IMM => { + self.emit_store_imm32(mem, OperandSize::S32, dst, insn.off as i32, insn.imm) + } + ebpf::ST_DW_IMM => { + self.emit_store_imm32(mem, OperandSize::S64, dst, insn.off as i32, insn.imm) + } + + // BPF_STX class + ebpf::ST_B_REG => self.emit_store(mem, OperandSize::S8, src, dst, insn.off as i32), + ebpf::ST_H_REG => self.emit_store(mem, OperandSize::S16, src, dst, insn.off as i32), + ebpf::ST_W_REG => self.emit_store(mem, OperandSize::S32, src, dst, insn.off as i32), + ebpf::ST_DW_REG => { + self.emit_store(mem, OperandSize::S64, src, dst, insn.off as i32) + } + ebpf::ST_W_XADD => unimplemented!(), + ebpf::ST_DW_XADD => unimplemented!(), + + // BPF_ALU class + ebpf::ADD32_IMM => self.emit_alu32_imm32(mem, 0x81, 0, dst, insn.imm), + ebpf::ADD32_REG => self.emit_alu32(mem, 0x01, src, dst), + ebpf::SUB32_IMM => self.emit_alu32_imm32(mem, 0x81, 5, dst, insn.imm), + ebpf::SUB32_REG => self.emit_alu32(mem, 0x29, src, dst), + ebpf::MUL32_IMM + | ebpf::MUL32_REG + | ebpf::DIV32_IMM + | ebpf::DIV32_REG + | ebpf::MOD32_IMM + | ebpf::MOD32_REG => { + self.emit_muldivmod(mem, insn_ptr as u16, insn.opc, src, dst, insn.imm) + } + ebpf::OR32_IMM => self.emit_alu32_imm32(mem, 0x81, 1, dst, insn.imm), + ebpf::OR32_REG => self.emit_alu32(mem, 0x09, src, dst), + ebpf::AND32_IMM => self.emit_alu32_imm32(mem, 0x81, 4, dst, insn.imm), + ebpf::AND32_REG => self.emit_alu32(mem, 0x21, src, dst), + ebpf::LSH32_IMM => self.emit_alu32_imm8(mem, 0xc1, 4, dst, insn.imm as i8), + ebpf::LSH32_REG => { + self.emit_mov(mem, src, RCX); + self.emit_alu32(mem, 0xd3, 4, dst); + } + ebpf::RSH32_IMM => self.emit_alu32_imm8(mem, 0xc1, 5, dst, insn.imm as i8), + ebpf::RSH32_REG => { + self.emit_mov(mem, src, RCX); + self.emit_alu32(mem, 0xd3, 5, dst); + } + ebpf::NEG32 => self.emit_alu32(mem, 0xf7, 3, dst), + ebpf::XOR32_IMM => self.emit_alu32_imm32(mem, 0x81, 6, dst, insn.imm), + ebpf::XOR32_REG => self.emit_alu32(mem, 0x31, src, dst), + ebpf::MOV32_IMM => self.emit_alu32_imm32(mem, 0xc7, 0, dst, insn.imm), + ebpf::MOV32_REG => self.emit_mov(mem, src, dst), + ebpf::ARSH32_IMM => self.emit_alu32_imm8(mem, 0xc1, 7, dst, insn.imm as i8), + ebpf::ARSH32_REG => { + self.emit_mov(mem, src, RCX); + self.emit_alu32(mem, 0xd3, 7, dst); + } + ebpf::LE => {} // No-op + ebpf::BE => { + match insn.imm { + 16 => { + // rol + self.emit1(mem, 0x66); // 16-bit override + self.emit_alu32_imm8(mem, 0xc1, 0, dst, 8); + // and + self.emit_alu32_imm32(mem, 0x81, 4, dst, 0xffff); + } + 32 | 64 => { + // bswap + let bit = match insn.imm { + 64 => 1, + _ => 0, + }; + self.emit_basic_rex(mem, bit, 0, dst); + self.emit1(mem, 0x0f); + self.emit1(mem, 0xc8 | (dst & 0b111)); + } + _ => unreachable!(), // Should have been caught by verifier + } + } + + // BPF_ALU64 class + ebpf::ADD64_IMM => self.emit_alu64_imm32(mem, 0x81, 0, dst, insn.imm), + ebpf::ADD64_REG => self.emit_alu64(mem, 0x01, src, dst), + ebpf::SUB64_IMM => self.emit_alu64_imm32(mem, 0x81, 5, dst, insn.imm), + ebpf::SUB64_REG => self.emit_alu64(mem, 0x29, src, dst), + ebpf::MUL64_IMM + | ebpf::MUL64_REG + | ebpf::DIV64_IMM + | ebpf::DIV64_REG + | ebpf::MOD64_IMM + | ebpf::MOD64_REG => { + self.emit_muldivmod(mem, insn_ptr as u16, insn.opc, src, dst, insn.imm) + } + ebpf::OR64_IMM => self.emit_alu64_imm32(mem, 0x81, 1, dst, insn.imm), + ebpf::OR64_REG => self.emit_alu64(mem, 0x09, src, dst), + ebpf::AND64_IMM => self.emit_alu64_imm32(mem, 0x81, 4, dst, insn.imm), + ebpf::AND64_REG => self.emit_alu64(mem, 0x21, src, dst), + ebpf::LSH64_IMM => self.emit_alu64_imm8(mem, 0xc1, 4, dst, insn.imm as i8), + ebpf::LSH64_REG => { + self.emit_mov(mem, src, RCX); + self.emit_alu64(mem, 0xd3, 4, dst); + } + ebpf::RSH64_IMM => self.emit_alu64_imm8(mem, 0xc1, 5, dst, insn.imm as i8), + ebpf::RSH64_REG => { + self.emit_mov(mem, src, RCX); + self.emit_alu64(mem, 0xd3, 5, dst); + } + ebpf::NEG64 => self.emit_alu64(mem, 0xf7, 3, dst), + ebpf::XOR64_IMM => self.emit_alu64_imm32(mem, 0x81, 6, dst, insn.imm), + ebpf::XOR64_REG => self.emit_alu64(mem, 0x31, src, dst), + ebpf::MOV64_IMM => self.emit_load_imm(mem, dst, insn.imm as i64), + ebpf::MOV64_REG => self.emit_mov(mem, src, dst), + ebpf::ARSH64_IMM => self.emit_alu64_imm8(mem, 0xc1, 7, dst, insn.imm as i8), + ebpf::ARSH64_REG => { + self.emit_mov(mem, src, RCX); + self.emit_alu64(mem, 0xd3, 7, dst); + } + + // BPF_JMP class + ebpf::JA => self.emit_jmp(mem, target_pc), + ebpf::JEQ_IMM => { + self.emit_cmp_imm32(mem, dst, insn.imm); + self.emit_jcc(mem, 0x84, target_pc); + } + ebpf::JEQ_REG => { + self.emit_cmp(mem, src, dst); + self.emit_jcc(mem, 0x84, target_pc); + } + ebpf::JGT_IMM => { + self.emit_cmp_imm32(mem, dst, insn.imm); + self.emit_jcc(mem, 0x87, target_pc); + } + ebpf::JGT_REG => { + self.emit_cmp(mem, src, dst); + self.emit_jcc(mem, 0x87, target_pc); + } + ebpf::JGE_IMM => { + self.emit_cmp_imm32(mem, dst, insn.imm); + self.emit_jcc(mem, 0x83, target_pc); + } + ebpf::JGE_REG => { + self.emit_cmp(mem, src, dst); + self.emit_jcc(mem, 0x83, target_pc); + } + ebpf::JLT_IMM => { + self.emit_cmp_imm32(mem, dst, insn.imm); + self.emit_jcc(mem, 0x82, target_pc); + } + ebpf::JLT_REG => { + self.emit_cmp(mem, src, dst); + self.emit_jcc(mem, 0x82, target_pc); + } + ebpf::JLE_IMM => { + self.emit_cmp_imm32(mem, dst, insn.imm); + self.emit_jcc(mem, 0x86, target_pc); + } + ebpf::JLE_REG => { + self.emit_cmp(mem, src, dst); + self.emit_jcc(mem, 0x86, target_pc); + } + ebpf::JSET_IMM => { + self.emit_alu64_imm32(mem, 0xf7, 0, dst, insn.imm); + self.emit_jcc(mem, 0x85, target_pc); + } + ebpf::JSET_REG => { + self.emit_alu64(mem, 0x85, src, dst); + self.emit_jcc(mem, 0x85, target_pc); + } + ebpf::JNE_IMM => { + self.emit_cmp_imm32(mem, dst, insn.imm); + self.emit_jcc(mem, 0x85, target_pc); + } + ebpf::JNE_REG => { + self.emit_cmp(mem, src, dst); + self.emit_jcc(mem, 0x85, target_pc); + } + ebpf::JSGT_IMM => { + self.emit_cmp_imm32(mem, dst, insn.imm); + self.emit_jcc(mem, 0x8f, target_pc); + } + ebpf::JSGT_REG => { + self.emit_cmp(mem, src, dst); + self.emit_jcc(mem, 0x8f, target_pc); + } + ebpf::JSGE_IMM => { + self.emit_cmp_imm32(mem, dst, insn.imm); + self.emit_jcc(mem, 0x8d, target_pc); + } + ebpf::JSGE_REG => { + self.emit_cmp(mem, src, dst); + self.emit_jcc(mem, 0x8d, target_pc); + } + ebpf::JSLT_IMM => { + self.emit_cmp_imm32(mem, dst, insn.imm); + self.emit_jcc(mem, 0x8c, target_pc); + } + ebpf::JSLT_REG => { + self.emit_cmp(mem, src, dst); + self.emit_jcc(mem, 0x8c, target_pc); + } + ebpf::JSLE_IMM => { + self.emit_cmp_imm32(mem, dst, insn.imm); + self.emit_jcc(mem, 0x8e, target_pc); + } + ebpf::JSLE_REG => { + self.emit_cmp(mem, src, dst); + self.emit_jcc(mem, 0x8e, target_pc); + } + + // BPF_JMP32 class + ebpf::JEQ_IMM32 => { + self.emit_cmp32_imm32(mem, dst, insn.imm); + self.emit_jcc(mem, 0x84, target_pc); + } + ebpf::JEQ_REG32 => { + self.emit_cmp32(mem, src, dst); + self.emit_jcc(mem, 0x84, target_pc); + } + ebpf::JGT_IMM32 => { + self.emit_cmp32_imm32(mem, dst, insn.imm); + self.emit_jcc(mem, 0x87, target_pc); + } + ebpf::JGT_REG32 => { + self.emit_cmp32(mem, src, dst); + self.emit_jcc(mem, 0x87, target_pc); + } + ebpf::JGE_IMM32 => { + self.emit_cmp32_imm32(mem, dst, insn.imm); + self.emit_jcc(mem, 0x83, target_pc); + } + ebpf::JGE_REG32 => { + self.emit_cmp32(mem, src, dst); + self.emit_jcc(mem, 0x83, target_pc); + } + ebpf::JLT_IMM32 => { + self.emit_cmp32_imm32(mem, dst, insn.imm); + self.emit_jcc(mem, 0x82, target_pc); + } + ebpf::JLT_REG32 => { + self.emit_cmp32(mem, src, dst); + self.emit_jcc(mem, 0x82, target_pc); + } + ebpf::JLE_IMM32 => { + self.emit_cmp32_imm32(mem, dst, insn.imm); + self.emit_jcc(mem, 0x86, target_pc); + } + ebpf::JLE_REG32 => { + self.emit_cmp32(mem, src, dst); + self.emit_jcc(mem, 0x86, target_pc); + } + ebpf::JSET_IMM32 => { + self.emit_alu32_imm32(mem, 0xf7, 0, dst, insn.imm); + self.emit_jcc(mem, 0x85, target_pc); + } + ebpf::JSET_REG32 => { + self.emit_alu32(mem, 0x85, src, dst); + self.emit_jcc(mem, 0x85, target_pc); + } + ebpf::JNE_IMM32 => { + self.emit_cmp32_imm32(mem, dst, insn.imm); + self.emit_jcc(mem, 0x85, target_pc); + } + ebpf::JNE_REG32 => { + self.emit_cmp32(mem, src, dst); + self.emit_jcc(mem, 0x85, target_pc); + } + ebpf::JSGT_IMM32 => { + self.emit_cmp32_imm32(mem, dst, insn.imm); + self.emit_jcc(mem, 0x8f, target_pc); + } + ebpf::JSGT_REG32 => { + self.emit_cmp32(mem, src, dst); + self.emit_jcc(mem, 0x8f, target_pc); + } + ebpf::JSGE_IMM32 => { + self.emit_cmp32_imm32(mem, dst, insn.imm); + self.emit_jcc(mem, 0x8d, target_pc); + } + ebpf::JSGE_REG32 => { + self.emit_cmp32(mem, src, dst); + self.emit_jcc(mem, 0x8d, target_pc); + } + ebpf::JSLT_IMM32 => { + self.emit_cmp32_imm32(mem, dst, insn.imm); + self.emit_jcc(mem, 0x8c, target_pc); + } + ebpf::JSLT_REG32 => { + self.emit_cmp32(mem, src, dst); + self.emit_jcc(mem, 0x8c, target_pc); + } + ebpf::JSLE_IMM32 => { + self.emit_cmp32_imm32(mem, dst, insn.imm); + self.emit_jcc(mem, 0x8e, target_pc); + } + ebpf::JSLE_REG32 => { + self.emit_cmp32(mem, src, dst); + self.emit_jcc(mem, 0x8e, target_pc); + } + + ebpf::CALL => { + // For JIT, helpers in use MUST be registered at compile time. They can be + // updated later, but not created after compiling (we need the address of the + // helper function in the JIT-compiled program). + if let Some(helper) = helpers.get(&(insn.imm as u32)) { + // We reserve RCX for shifts + self.emit_mov(mem, R9, RCX); + self.emit_call(mem, *helper as usize); + } else { + Err(Error::new( + ErrorKind::Other, + format!( + "[JIT] Error: unknown helper function (id: {:#x})", + insn.imm as u32 + ), + ))?; + }; + } + ebpf::TAIL_CALL => { + unimplemented!() + } + ebpf::EXIT => { + if insn_ptr != prog.len() / ebpf::INSN_SIZE - 1 { + self.emit_jmp(mem, TARGET_PC_EXIT); + }; + } + + _ => { + Err(Error::new( + ErrorKind::Other, + format!( + "[JIT] Error: unknown eBPF opcode {:#2x} (insn #{insn_ptr:?})", + insn.opc + ), + ))?; + } + } + + insn_ptr += 1; + } + + // Epilogue + self.set_anchor(mem, TARGET_PC_EXIT); + + // Move register 0 into rax + if map_register(0) != RAX { + self.emit_mov(mem, map_register(0), RAX); + } + + // Deallocate stack space + self.emit_alu64_imm32(mem, 0x81, 0, RSP, ebpf::STACK_SIZE as i32); + + self.emit_pop(mem, R15); + self.emit_pop(mem, R14); + self.emit_pop(mem, R13); + self.emit_pop(mem, RBX); + self.emit_pop(mem, RBP); + + self.emit1(mem, 0xc3); // ret + + Ok(()) + } + + fn resolve_jumps(&mut self, mem: &mut JitMemory) -> Result<(), Error> { + for jump in &self.jumps { + let target_loc = match self.special_targets.get(&jump.target_pc) { + Some(target) => *target, + None => self.pc_locs[jump.target_pc as usize], + }; + + // Assumes jump offset is at end of instruction + unsafe { + let offset_loc = jump.offset_loc as i32 + std::mem::size_of::() as i32; + let rel = &(target_loc as i32 - offset_loc) as *const i32; + + let offset_ptr = mem.contents.as_ptr().add(jump.offset_loc); + + libc::memcpy( + offset_ptr as *mut libc::c_void, + rel as *const libc::c_void, + std::mem::size_of::(), + ); + } + } + Ok(()) + } +} // impl JitCompiler + +pub struct JitMemory<'a> { + contents: &'a mut [u8], + offset: usize, +} + +impl<'a> JitMemory<'a> { + pub fn new( + prog: &[u8], + helpers: &HashMap, + use_mbuff: bool, + update_data_ptr: bool, + ) -> Result, Error> { + let contents: &mut [u8]; + let mut raw: mem::MaybeUninit<*mut libc::c_void> = mem::MaybeUninit::uninit(); + unsafe { + let size = NUM_PAGES * PAGE_SIZE; + libc::posix_memalign(raw.as_mut_ptr(), PAGE_SIZE, size); + libc::mprotect( + *raw.as_mut_ptr(), + size, + libc::PROT_EXEC | libc::PROT_READ | libc::PROT_WRITE, + ); + std::ptr::write_bytes(*raw.as_mut_ptr(), 0xc3, size); // for now, prepopulate with 'RET' calls + contents = + std::slice::from_raw_parts_mut(*raw.as_mut_ptr() as *mut u8, NUM_PAGES * PAGE_SIZE); + raw.assume_init(); + } + + let mut mem = JitMemory { + contents, + offset: 0, + }; + + let mut jit = JitCompiler::new(); + jit.jit_compile(&mut mem, prog, use_mbuff, update_data_ptr, helpers)?; + jit.resolve_jumps(&mut mem)?; + + Ok(mem) + } + + pub fn get_prog(&self) -> MachineCode { + unsafe { mem::transmute(self.contents.as_ptr()) } + } +} + +impl<'a> Index for JitMemory<'a> { + type Output = u8; + + fn index(&self, _index: usize) -> &u8 { + &self.contents[_index] + } +} + +impl<'a> IndexMut for JitMemory<'a> { + fn index_mut(&mut self, _index: usize) -> &mut u8 { + &mut self.contents[_index] + } +} + +impl<'a> Drop for JitMemory<'a> { + fn drop(&mut self) { + unsafe { + libc::free(self.contents.as_mut_ptr() as *mut libc::c_void); + } + } +} + +impl<'a> std::fmt::Debug for JitMemory<'a> { + fn fmt(&self, fmt: &mut Formatter) -> Result<(), FormatterError> { + fmt.write_str("JIT contents: [")?; + fmt.write_str(" ] | ")?; + fmt.debug_struct("JIT memory") + .field("offset", &self.offset) + .finish() + } +} diff --git a/kernel/crates/rbpf/src/lib.rs b/kernel/crates/rbpf/src/lib.rs new file mode 100644 index 000000000..0d4fa2839 --- /dev/null +++ b/kernel/crates/rbpf/src/lib.rs @@ -0,0 +1,1782 @@ +// SPDX-License-Identifier: (Apache-2.0 OR MIT) +// Derived from uBPF +// Copyright 2016 6WIND S.A. +// Copyright 2023 Isovalent, Inc. + +//! Virtual machine and JIT compiler for eBPF programs. +#![doc( + html_logo_url = "https://raw.githubusercontent.com/qmonnet/rbpf/main/misc/rbpf.png", + html_favicon_url = "https://raw.githubusercontent.com/qmonnet/rbpf/main/misc/rbpf.ico" +)] +#![warn(missing_docs)] +// There are unused mut warnings due to unsafe code. +#![allow(unused_mut)] +// Allows old-style clippy +#![allow(renamed_and_removed_lints)] +#![cfg_attr( + clippy, + allow( + redundant_field_names, + single_match, + cast_lossless, + doc_markdown, + match_same_arms, + unreadable_literal + ) +)] +// Configures the crate to be `no_std` when `std` feature is disabled. +#![cfg_attr(not(feature = "std"), no_std)] +extern crate alloc; +use alloc::{collections::BTreeMap, format, vec, vec::Vec}; + +use byteorder::{ByteOrder, LittleEndian}; + +type HashMap = BTreeMap; +#[cfg(feature = "cranelift")] +type HashSet = alloc::collections::BTreeSet; +mod asm_parser; +pub mod assembler; +#[cfg(feature = "cranelift")] +mod cranelift; +pub mod disassembler; +pub mod ebpf; +pub mod helpers; +pub mod insn_builder; +mod interpreter; +#[cfg(all(not(windows), feature = "std"))] +mod jit; +#[cfg(not(feature = "std"))] +mod no_std_error; +mod stack; +mod verifier; + +#[cfg(feature = "std")] +pub use std::io::{Error, ErrorKind}; + +/// In no_std we use a custom implementation of the error which acts as a +/// replacement for the io Error. +#[cfg(not(feature = "std"))] +pub use crate::no_std_error::{Error, ErrorKind}; + +/// eBPF verification function that returns an error if the program does not meet its requirements. +/// +/// Some examples of things the verifier may reject the program for: +/// +/// - Program does not terminate. +/// - Unknown instructions. +/// - Bad formed instruction. +/// - Unknown eBPF helper index. +pub type Verifier = fn(prog: &[u8]) -> Result<(), Error>; + +/// eBPF helper function. +pub type Helper = fn(u64, u64, u64, u64, u64) -> u64; + +// A metadata buffer with two offset indications. It can be used in one kind of eBPF VM to simulate +// the use of a metadata buffer each time the program is executed, without the user having to +// actually handle it. The offsets are used to tell the VM where in the buffer the pointers to +// packet data start and end should be stored each time the program is run on a new packet. +struct MetaBuff { + data_offset: usize, + data_end_offset: usize, + buffer: Vec, +} + +/// A virtual machine to run eBPF program. This kind of VM is used for programs expecting to work +/// on a metadata buffer containing pointers to packet data. +/// +/// # Examples +/// +/// ``` +/// let prog = &[ +/// 0x79, 0x11, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, // Load mem from mbuff at offset 8 into R1. +/// 0x69, 0x10, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, // ldhx r1[2], r0 +/// 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 // exit +/// ]; +/// let mem = &mut [ +/// 0xaa, 0xbb, 0x11, 0x22, 0xcc, 0xdd +/// ]; +/// +/// // Just for the example we create our metadata buffer from scratch, and we store the pointers +/// // to packet data start and end in it. +/// let mut mbuff = [0u8; 32]; +/// unsafe { +/// let mut data = mbuff.as_ptr().offset(8) as *mut u64; +/// let mut data_end = mbuff.as_ptr().offset(24) as *mut u64; +/// *data = mem.as_ptr() as u64; +/// *data_end = mem.as_ptr() as u64 + mem.len() as u64; +/// } +/// +/// // Instantiate a VM. +/// let mut vm = rbpf::EbpfVmMbuff::new(Some(prog)).unwrap(); +/// +/// // Provide both a reference to the packet data, and to the metadata buffer. +/// let res = vm.execute_program(mem, &mut mbuff).unwrap(); +/// assert_eq!(res, 0x2211); +/// ``` +pub struct EbpfVmMbuff<'a> { + prog: Option<&'a [u8]>, + verifier: Verifier, + #[cfg(all(not(windows), feature = "std"))] + jit: Option>, + #[cfg(feature = "cranelift")] + cranelift_prog: Option, + helpers: HashMap, +} + +impl<'a> EbpfVmMbuff<'a> { + /// Create a new virtual machine instance, and load an eBPF program into that instance. + /// When attempting to load the program, it passes through a simple verifier. + /// + /// # Examples + /// + /// ``` + /// let prog = &[ + /// 0x79, 0x11, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, // Load mem from mbuff into R1. + /// 0x69, 0x10, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, // ldhx r1[2], r0 + /// 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 // exit + /// ]; + /// + /// // Instantiate a VM. + /// let mut vm = rbpf::EbpfVmMbuff::new(Some(prog)).unwrap(); + /// ``` + pub fn new(prog: Option<&'a [u8]>) -> Result, Error> { + if let Some(prog) = prog { + verifier::check(prog)?; + } + + Ok(EbpfVmMbuff { + prog, + verifier: verifier::check, + #[cfg(all(not(windows), feature = "std"))] + jit: None, + #[cfg(feature = "cranelift")] + cranelift_prog: None, + helpers: HashMap::new(), + }) + } + + /// Load a new eBPF program into the virtual machine instance. + /// + /// # Examples + /// + /// ``` + /// let prog1 = &[ + /// 0xb7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // mov r0, 0 + /// 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 // exit + /// ]; + /// let prog2 = &[ + /// 0x79, 0x11, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, // Load mem from mbuff into R1. + /// 0x69, 0x10, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, // ldhx r1[2], r0 + /// 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 // exit + /// ]; + /// + /// // Instantiate a VM. + /// let mut vm = rbpf::EbpfVmMbuff::new(Some(prog1)).unwrap(); + /// vm.set_program(prog2).unwrap(); + /// ``` + pub fn set_program(&mut self, prog: &'a [u8]) -> Result<(), Error> { + (self.verifier)(prog)?; + self.prog = Some(prog); + Ok(()) + } + + /// Set a new verifier function. The function should return an `Error` if the program should be + /// rejected by the virtual machine. If a program has been loaded to the VM already, the + /// verifier is immediately run. + /// + /// # Examples + /// + /// ``` + /// use rbpf::{Error, ErrorKind}; + /// use rbpf::ebpf; + /// + /// // Define a simple verifier function. + /// fn verifier(prog: &[u8]) -> Result<(), Error> { + /// let last_insn = ebpf::get_insn(prog, (prog.len() / ebpf::INSN_SIZE) - 1); + /// if last_insn.opc != ebpf::EXIT { + /// return Err(Error::new(ErrorKind::Other, + /// "[Verifier] Error: program does not end with “EXIT” instruction")); + /// } + /// Ok(()) + /// } + /// + /// let prog1 = &[ + /// 0xb7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // mov r0, 0 + /// 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 // exit + /// ]; + /// + /// // Instantiate a VM. + /// let mut vm = rbpf::EbpfVmMbuff::new(Some(prog1)).unwrap(); + /// // Change the verifier. + /// vm.set_verifier(verifier).unwrap(); + /// ``` + pub fn set_verifier(&mut self, verifier: Verifier) -> Result<(), Error> { + if let Some(prog) = self.prog { + verifier(prog)?; + } + self.verifier = verifier; + Ok(()) + } + + /// Register a built-in or user-defined helper function in order to use it later from within + /// the eBPF program. The helper is registered into a hashmap, so the `key` can be any `u32`. + /// + /// If using JIT-compiled eBPF programs, be sure to register all helpers before compiling the + /// program. You should be able to change registered helpers after compiling, but not to add + /// new ones (i.e. with new keys). + /// + /// # Examples + /// + /// ``` + /// use rbpf::helpers; + /// + /// // This program was compiled with clang, from a C program containing the following single + /// // instruction: `return bpf_trace_printk("foo %c %c %c\n", 10, 1, 2, 3);` + /// let prog = &[ + /// 0x18, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // load 0 as u64 into r1 (That would be + /// 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // replaced by tc by the address of + /// // the format string, in the .map + /// // section of the ELF file). + /// 0xb7, 0x02, 0x00, 0x00, 0x0a, 0x00, 0x00, 0x00, // mov r2, 10 + /// 0xb7, 0x03, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, // mov r3, 1 + /// 0xb7, 0x04, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, // mov r4, 2 + /// 0xb7, 0x05, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, // mov r5, 3 + /// 0x85, 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, 0x00, // call helper with key 6 + /// 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 // exit + /// ]; + /// + /// // Instantiate a VM. + /// let mut vm = rbpf::EbpfVmMbuff::new(Some(prog)).unwrap(); + /// + /// // Register a helper. + /// // On running the program this helper will print the content of registers r3, r4 and r5 to + /// // standard output. + /// # #[cfg(feature = "std")] + /// vm.register_helper(6, helpers::bpf_trace_printf).unwrap(); + /// ``` + pub fn register_helper(&mut self, key: u32, function: Helper) -> Result<(), Error> { + self.helpers.insert(key, function); + Ok(()) + } + + /// Execute the program loaded, with the given packet data and metadata buffer. + /// + /// If the program is made to be compatible with Linux kernel, it is expected to load the + /// address of the beginning and of the end of the memory area used for packet data from the + /// metadata buffer, at some appointed offsets. It is up to the user to ensure that these + /// pointers are correctly stored in the buffer. + /// + /// # Examples + /// + /// ``` + /// let prog = &[ + /// 0x79, 0x11, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, // Load mem from mbuff into R1. + /// 0x69, 0x10, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, // ldhx r1[2], r0 + /// 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 // exit + /// ]; + /// let mem = &mut [ + /// 0xaa, 0xbb, 0x11, 0x22, 0xcc, 0xdd + /// ]; + /// + /// // Just for the example we create our metadata buffer from scratch, and we store the + /// // pointers to packet data start and end in it. + /// let mut mbuff = [0u8; 32]; + /// unsafe { + /// let mut data = mbuff.as_ptr().offset(8) as *mut u64; + /// let mut data_end = mbuff.as_ptr().offset(24) as *mut u64; + /// *data = mem.as_ptr() as u64; + /// *data_end = mem.as_ptr() as u64 + mem.len() as u64; + /// } + /// + /// // Instantiate a VM. + /// let mut vm = rbpf::EbpfVmMbuff::new(Some(prog)).unwrap(); + /// + /// // Provide both a reference to the packet data, and to the metadata buffer. + /// let res = vm.execute_program(mem, &mut mbuff).unwrap(); + /// assert_eq!(res, 0x2211); + /// ``` + pub fn execute_program(&self, mem: &[u8], mbuff: &[u8]) -> Result { + interpreter::execute_program(self.prog, mem, mbuff, &self.helpers) + } + + /// JIT-compile the loaded program. No argument required for this. + /// + /// If using helper functions, be sure to register them into the VM before calling this + /// function. + /// + /// # Examples + /// + /// ``` + /// let prog = &[ + /// 0x79, 0x11, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, // Load mem from mbuff into R1. + /// 0x69, 0x10, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, // ldhx r1[2], r0 + /// 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 // exit + /// ]; + /// + /// // Instantiate a VM. + /// let mut vm = rbpf::EbpfVmMbuff::new(Some(prog)).unwrap(); + /// + /// vm.jit_compile(); + /// ``` + #[cfg(all(not(windows), feature = "std"))] + pub fn jit_compile(&mut self) -> Result<(), Error> { + let prog = match self.prog { + Some(prog) => prog, + None => Err(Error::new( + ErrorKind::Other, + "Error: No program set, call prog_set() to load one", + ))?, + }; + self.jit = Some(jit::JitMemory::new(prog, &self.helpers, true, false)?); + Ok(()) + } + + /// Execute the previously JIT-compiled program, with the given packet data and metadata + /// buffer, in a manner very similar to `execute_program()`. + /// + /// If the program is made to be compatible with Linux kernel, it is expected to load the + /// address of the beginning and of the end of the memory area used for packet data from the + /// metadata buffer, at some appointed offsets. It is up to the user to ensure that these + /// pointers are correctly stored in the buffer. + /// + /// # Safety + /// + /// **WARNING:** JIT-compiled assembly code is not safe, in particular there is no runtime + /// check for memory access; so if the eBPF program attempts erroneous accesses, this may end + /// very bad (program may segfault). It may be wise to check that the program works with the + /// interpreter before running the JIT-compiled version of it. + /// + /// For this reason the function should be called from within an `unsafe` bloc. + /// + /// # Examples + /// + /// ``` + /// let prog = &[ + /// 0x79, 0x11, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, // Load mem from mbuff into r1. + /// 0x69, 0x10, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, // ldhx r1[2], r0 + /// 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 // exit + /// ]; + /// let mem = &mut [ + /// 0xaa, 0xbb, 0x11, 0x22, 0xcc, 0xdd + /// ]; + /// + /// // Just for the example we create our metadata buffer from scratch, and we store the + /// // pointers to packet data start and end in it. + /// let mut mbuff = [0u8; 32]; + /// unsafe { + /// let mut data = mbuff.as_ptr().offset(8) as *mut u64; + /// let mut data_end = mbuff.as_ptr().offset(24) as *mut u64; + /// *data = mem.as_ptr() as u64; + /// *data_end = mem.as_ptr() as u64 + mem.len() as u64; + /// } + /// + /// // Instantiate a VM. + /// let mut vm = rbpf::EbpfVmMbuff::new(Some(prog)).unwrap(); + /// + /// # #[cfg(all(not(windows), feature = "std"))] + /// vm.jit_compile(); + /// + /// // Provide both a reference to the packet data, and to the metadata buffer. + /// # #[cfg(all(not(windows), feature = "std"))] + /// unsafe { + /// let res = vm.execute_program_jit(mem, &mut mbuff).unwrap(); + /// assert_eq!(res, 0x2211); + /// } + /// ``` + #[cfg(all(not(windows), feature = "std"))] + pub unsafe fn execute_program_jit( + &self, + mem: &mut [u8], + mbuff: &'a mut [u8], + ) -> Result { + // If packet data is empty, do not send the address of an empty slice; send a null pointer + // as first argument instead, as this is uBPF's behavior (empty packet should not happen + // in the kernel; anyway the verifier would prevent the use of uninitialized registers). + // See `mul_loop` test. + let mem_ptr = match mem.len() { + 0 => std::ptr::null_mut(), + _ => mem.as_ptr() as *mut u8, + }; + // The last two arguments are not used in this function. They would be used if there was a + // need to indicate to the JIT at which offset in the mbuff mem_ptr and mem_ptr + mem.len() + // should be stored; this is what happens with struct EbpfVmFixedMbuff. + match &self.jit { + Some(jit) => Ok(jit.get_prog()( + mbuff.as_ptr() as *mut u8, + mbuff.len(), + mem_ptr, + mem.len(), + 0, + 0, + )), + None => Err(Error::new( + ErrorKind::Other, + "Error: program has not been JIT-compiled", + )), + } + } + + /// Compile the loaded program using the Cranelift JIT. + /// + /// If using helper functions, be sure to register them into the VM before calling this + /// function. + /// + /// # Examples + /// + /// ``` + /// let prog = &[ + /// 0x79, 0x11, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, // Load mem from mbuff into R1. + /// 0x69, 0x10, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, // ldhx r1[2], r0 + /// 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 // exit + /// ]; + /// + /// // Instantiate a VM. + /// let mut vm = rbpf::EbpfVmMbuff::new(Some(prog)).unwrap(); + /// + /// vm.cranelift_compile(); + /// ``` + #[cfg(feature = "cranelift")] + pub fn cranelift_compile(&mut self) -> Result<(), Error> { + use crate::cranelift::CraneliftCompiler; + + let prog = match self.prog { + Some(prog) => prog, + None => Err(Error::new( + ErrorKind::Other, + "Error: No program set, call prog_set() to load one", + ))?, + }; + + let mut compiler = CraneliftCompiler::new(self.helpers.clone()); + let program = compiler.compile_function(prog)?; + + self.cranelift_prog = Some(program); + Ok(()) + } + + /// Execute the previously compiled program, with the given packet data and metadata + /// buffer, in a manner very similar to `execute_program()`. + /// + /// If the program is made to be compatible with Linux kernel, it is expected to load the + /// address of the beginning and of the end of the memory area used for packet data from the + /// metadata buffer, at some appointed offsets. It is up to the user to ensure that these + /// pointers are correctly stored in the buffer. + /// + /// + /// # Examples + /// + /// ``` + /// let prog = &[ + /// 0x79, 0x11, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, // Load mem from mbuff into r1. + /// 0x69, 0x10, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, // ldhx r1[2], r0 + /// 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 // exit + /// ]; + /// let mem = &mut [ + /// 0xaa, 0xbb, 0x11, 0x22, 0xcc, 0xdd + /// ]; + /// + /// // Just for the example we create our metadata buffer from scratch, and we store the + /// // pointers to packet data start and end in it. + /// let mut mbuff = [0u8; 32]; + /// unsafe { + /// let mut data = mbuff.as_ptr().offset(8) as *mut u64; + /// let mut data_end = mbuff.as_ptr().offset(24) as *mut u64; + /// *data = mem.as_ptr() as u64; + /// *data_end = mem.as_ptr() as u64 + mem.len() as u64; + /// } + /// + /// // Instantiate a VM. + /// let mut vm = rbpf::EbpfVmMbuff::new(Some(prog)).unwrap(); + /// + /// vm.cranelift_compile(); + /// + /// // Provide both a reference to the packet data, and to the metadata buffer. + /// let res = vm.execute_program_cranelift(mem, &mut mbuff).unwrap(); + /// assert_eq!(res, 0x2211); + /// ``` + #[cfg(feature = "cranelift")] + pub fn execute_program_cranelift( + &self, + mem: &mut [u8], + mbuff: &'a mut [u8], + ) -> Result { + // If packet data is empty, do not send the address of an empty slice; send a null pointer + // as first argument instead, as this is uBPF's behavior (empty packet should not happen + // in the kernel; anyway the verifier would prevent the use of uninitialized registers). + // See `mul_loop` test. + let mem_ptr = match mem.len() { + 0 => core::ptr::null_mut(), + _ => mem.as_ptr() as *mut u8, + }; + + // The last two arguments are not used in this function. They would be used if there was a + // need to indicate to the JIT at which offset in the mbuff mem_ptr and mem_ptr + mem.len() + // should be stored; this is what happens with struct EbpfVmFixedMbuff. + match &self.cranelift_prog { + Some(prog) => { + Ok(prog.execute(mem_ptr, mem.len(), mbuff.as_ptr() as *mut u8, mbuff.len())) + } + None => Err(Error::new( + ErrorKind::Other, + "Error: program has not been compiled with cranelift", + )), + } + } +} + +/// A virtual machine to run eBPF program. This kind of VM is used for programs expecting to work +/// on a metadata buffer containing pointers to packet data, but it internally handles the buffer +/// so as to save the effort to manually handle the metadata buffer for the user. +/// +/// This struct implements a static internal buffer that is passed to the program. The user has to +/// indicate the offset values at which the eBPF program expects to find the start and the end of +/// packet data in the buffer. On calling the `execute_program()` or `execute_program_jit()` functions, the +/// struct automatically updates the addresses in this static buffer, at the appointed offsets, for +/// the start and the end of the packet data the program is called upon. +/// +/// # Examples +/// +/// This was compiled with clang from the following program, in C: +/// +/// ```c +/// #include +/// #include "path/to/linux/samples/bpf/bpf_helpers.h" +/// +/// SEC(".classifier") +/// int classifier(struct __sk_buff *skb) +/// { +/// void *data = (void *)(long)skb->data; +/// void *data_end = (void *)(long)skb->data_end; +/// +/// // Check program is long enough. +/// if (data + 5 > data_end) +/// return 0; +/// +/// return *((char *)data + 5); +/// } +/// ``` +/// +/// Some small modifications have been brought to have it work, see comments. +/// +/// ``` +/// let prog = &[ +/// 0xb7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // mov r0, 0 +/// // Here opcode 0x61 had to be replace by 0x79 so as to load a 8-bytes long address. +/// // Also, offset 0x4c had to be replace with e.g. 0x40 so as to prevent the two pointers +/// // from overlapping in the buffer. +/// 0x79, 0x12, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, // load pointer to mem from r1[0x40] to r2 +/// 0x07, 0x02, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, // add r2, 5 +/// // Here opcode 0x61 had to be replace by 0x79 so as to load a 8-bytes long address. +/// 0x79, 0x11, 0x50, 0x00, 0x00, 0x00, 0x00, 0x00, // load ptr to mem_end from r1[0x50] to r1 +/// 0x2d, 0x12, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, // if r2 > r1 skip 3 instructions +/// 0x71, 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // load r2 (= *(mem + 5)) into r0 +/// 0x67, 0x00, 0x00, 0x00, 0x38, 0x00, 0x00, 0x00, // r0 >>= 56 +/// 0xc7, 0x00, 0x00, 0x00, 0x38, 0x00, 0x00, 0x00, // r0 <<= 56 (arsh) extend byte sign to u64 +/// 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 // exit +/// ]; +/// let mem1 = &mut [ +/// 0xaa, 0xbb, 0x11, 0x22, 0xcc, 0xdd +/// ]; +/// let mem2 = &mut [ +/// 0xaa, 0xbb, 0x11, 0x22, 0xcc, 0x27 +/// ]; +/// +/// // Instantiate a VM. Note that we provide the start and end offsets for mem pointers. +/// let mut vm = rbpf::EbpfVmFixedMbuff::new(Some(prog), 0x40, 0x50).unwrap(); +/// +/// // Provide only a reference to the packet data. We do not manage the metadata buffer. +/// let res = vm.execute_program(mem1).unwrap(); +/// assert_eq!(res, 0xffffffffffffffdd); +/// +/// let res = vm.execute_program(mem2).unwrap(); +/// assert_eq!(res, 0x27); +/// ``` +pub struct EbpfVmFixedMbuff<'a> { + parent: EbpfVmMbuff<'a>, + mbuff: MetaBuff, +} + +impl<'a> EbpfVmFixedMbuff<'a> { + /// Create a new virtual machine instance, and load an eBPF program into that instance. + /// When attempting to load the program, it passes through a simple verifier. + /// + /// # Examples + /// + /// ``` + /// let prog = &[ + /// 0xb7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // mov r0, 0 + /// 0x79, 0x12, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, // load mem from r1[0x40] to r2 + /// 0x07, 0x02, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, // add r2, 5 + /// 0x79, 0x11, 0x50, 0x00, 0x00, 0x00, 0x00, 0x00, // load mem_end from r1[0x50] to r1 + /// 0x2d, 0x12, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, // if r2 > r1 skip 3 instructions + /// 0x71, 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // load r2 (= *(mem + 5)) into r0 + /// 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 // exit + /// ]; + /// + /// // Instantiate a VM. Note that we provide the start and end offsets for mem pointers. + /// let mut vm = rbpf::EbpfVmFixedMbuff::new(Some(prog), 0x40, 0x50).unwrap(); + /// ``` + pub fn new( + prog: Option<&'a [u8]>, + data_offset: usize, + data_end_offset: usize, + ) -> Result, Error> { + let parent = EbpfVmMbuff::new(prog)?; + let get_buff_len = |x: usize, y: usize| if x >= y { x + 8 } else { y + 8 }; + let buffer = vec![0u8; get_buff_len(data_offset, data_end_offset)]; + let mbuff = MetaBuff { + data_offset, + data_end_offset, + buffer, + }; + Ok(EbpfVmFixedMbuff { parent, mbuff }) + } + + /// Load a new eBPF program into the virtual machine instance. + /// + /// At the same time, load new offsets for storing pointers to start and end of packet data in + /// the internal metadata buffer. + /// + /// # Examples + /// + /// ``` + /// let prog1 = &[ + /// 0xb7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // mov r0, 0 + /// 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 // exit + /// ]; + /// let prog2 = &[ + /// 0xb7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // mov r0, 0 + /// 0x79, 0x12, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, // load mem from r1[0x40] to r2 + /// 0x07, 0x02, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, // add r2, 5 + /// 0x79, 0x11, 0x50, 0x00, 0x00, 0x00, 0x00, 0x00, // load mem_end from r1[0x50] to r1 + /// 0x2d, 0x12, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, // if r2 > r1 skip 3 instructions + /// 0x71, 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // load r2 (= *(mem + 5)) into r0 + /// 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 // exit + /// ]; + /// + /// let mem = &mut [ + /// 0xaa, 0xbb, 0x11, 0x22, 0xcc, 0x27, + /// ]; + /// + /// let mut vm = rbpf::EbpfVmFixedMbuff::new(Some(prog1), 0, 0).unwrap(); + /// vm.set_program(prog2, 0x40, 0x50); + /// + /// let res = vm.execute_program(mem).unwrap(); + /// assert_eq!(res, 0x27); + /// ``` + pub fn set_program( + &mut self, + prog: &'a [u8], + data_offset: usize, + data_end_offset: usize, + ) -> Result<(), Error> { + let get_buff_len = |x: usize, y: usize| if x >= y { x + 8 } else { y + 8 }; + let buffer = vec![0u8; get_buff_len(data_offset, data_end_offset)]; + self.mbuff.buffer = buffer; + self.mbuff.data_offset = data_offset; + self.mbuff.data_end_offset = data_end_offset; + self.parent.set_program(prog)?; + Ok(()) + } + + /// Set a new verifier function. The function should return an `Error` if the program should be + /// rejected by the virtual machine. If a program has been loaded to the VM already, the + /// verifier is immediately run. + /// + /// # Examples + /// + /// ``` + /// use rbpf::{Error, ErrorKind}; + /// use rbpf::ebpf; + /// + /// // Define a simple verifier function. + /// fn verifier(prog: &[u8]) -> Result<(), Error> { + /// let last_insn = ebpf::get_insn(prog, (prog.len() / ebpf::INSN_SIZE) - 1); + /// if last_insn.opc != ebpf::EXIT { + /// return Err(Error::new(ErrorKind::Other, + /// "[Verifier] Error: program does not end with “EXIT” instruction")); + /// } + /// Ok(()) + /// } + /// + /// let prog1 = &[ + /// 0xb7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // mov r0, 0 + /// 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 // exit + /// ]; + /// + /// // Instantiate a VM. + /// let mut vm = rbpf::EbpfVmMbuff::new(Some(prog1)).unwrap(); + /// // Change the verifier. + /// vm.set_verifier(verifier).unwrap(); + /// ``` + pub fn set_verifier(&mut self, verifier: Verifier) -> Result<(), Error> { + self.parent.set_verifier(verifier) + } + + /// Register a built-in or user-defined helper function in order to use it later from within + /// the eBPF program. The helper is registered into a hashmap, so the `key` can be any `u32`. + /// + /// If using JIT-compiled eBPF programs, be sure to register all helpers before compiling the + /// program. You should be able to change registered helpers after compiling, but not to add + /// new ones (i.e. with new keys). + /// + /// # Examples + /// + /// ``` + /// #[cfg(feature = "std")] { + /// use rbpf::helpers; + /// + /// // This program was compiled with clang, from a C program containing the following single + /// // instruction: `return bpf_trace_printk("foo %c %c %c\n", 10, 1, 2, 3);` + /// let prog = &[ + /// 0xb7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // mov r0, 0 + /// 0x79, 0x12, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, // load mem from r1[0x40] to r2 + /// 0x07, 0x02, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, // add r2, 5 + /// 0x79, 0x11, 0x50, 0x00, 0x00, 0x00, 0x00, 0x00, // load mem_end from r1[0x50] to r1 + /// 0x2d, 0x12, 0x06, 0x00, 0x00, 0x00, 0x00, 0x00, // if r2 > r1 skip 6 instructions + /// 0x71, 0x21, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // load r2 (= *(mem + 5)) into r1 + /// 0xb7, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // mov r2, 0 + /// 0xb7, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // mov r3, 0 + /// 0xb7, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // mov r4, 0 + /// 0xb7, 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // mov r5, 0 + /// 0x85, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, // call helper with key 1 + /// 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 // exit + /// ]; + /// + /// let mem = &mut [ + /// 0xaa, 0xbb, 0x11, 0x22, 0xcc, 0x09, + /// ]; + /// + /// // Instantiate a VM. + /// let mut vm = rbpf::EbpfVmFixedMbuff::new(Some(prog), 0x40, 0x50).unwrap(); + /// + /// // Register a helper. This helper will store the result of the square root of r1 into r0. + /// vm.register_helper(1, helpers::sqrti); + /// + /// let res = vm.execute_program(mem).unwrap(); + /// assert_eq!(res, 3); + /// } + /// ``` + pub fn register_helper( + &mut self, + key: u32, + function: fn(u64, u64, u64, u64, u64) -> u64, + ) -> Result<(), Error> { + self.parent.register_helper(key, function) + } + + /// Execute the program loaded, with the given packet data. + /// + /// If the program is made to be compatible with Linux kernel, it is expected to load the + /// address of the beginning and of the end of the memory area used for packet data from some + /// metadata buffer, which in the case of this VM is handled internally. The offsets at which + /// the addresses should be placed should have be set at the creation of the VM. + /// + /// # Examples + /// + /// ``` + /// let prog = &[ + /// 0xb7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // mov r0, 0 + /// 0x79, 0x12, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, // load mem from r1[0x40] to r2 + /// 0x07, 0x02, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, // add r2, 5 + /// 0x79, 0x11, 0x50, 0x00, 0x00, 0x00, 0x00, 0x00, // load mem_end from r1[0x50] to r1 + /// 0x2d, 0x12, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, // if r2 > r1 skip 3 instructions + /// 0x71, 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // load r2 (= *(mem + 5)) into r0 + /// 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 // exit + /// ]; + /// let mem = &mut [ + /// 0xaa, 0xbb, 0x11, 0x22, 0xcc, 0xdd + /// ]; + /// + /// // Instantiate a VM. Note that we provide the start and end offsets for mem pointers. + /// let mut vm = rbpf::EbpfVmFixedMbuff::new(Some(prog), 0x40, 0x50).unwrap(); + /// + /// // Provide only a reference to the packet data. We do not manage the metadata buffer. + /// let res = vm.execute_program(mem).unwrap(); + /// assert_eq!(res, 0xdd); + /// ``` + pub fn execute_program(&mut self, mem: &'a mut [u8]) -> Result { + let l = self.mbuff.buffer.len(); + // Can this ever happen? Probably not, should be ensured at mbuff creation. + if self.mbuff.data_offset + 8 > l || self.mbuff.data_end_offset + 8 > l { + Err(Error::new(ErrorKind::Other, format!("Error: buffer too small ({:?}), cannot use data_offset {:?} and data_end_offset {:?}", + l, self.mbuff.data_offset, self.mbuff.data_end_offset)))?; + } + LittleEndian::write_u64( + &mut self.mbuff.buffer[(self.mbuff.data_offset)..], + mem.as_ptr() as u64, + ); + LittleEndian::write_u64( + &mut self.mbuff.buffer[(self.mbuff.data_end_offset)..], + mem.as_ptr() as u64 + mem.len() as u64, + ); + self.parent.execute_program(mem, &self.mbuff.buffer) + } + + /// JIT-compile the loaded program. No argument required for this. + /// + /// If using helper functions, be sure to register them into the VM before calling this + /// function. + /// + /// # Examples + /// + /// ``` + /// let prog = &[ + /// 0xb7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // mov r0, 0 + /// 0x79, 0x12, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, // load mem from r1[0x40] to r2 + /// 0x07, 0x02, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, // add r2, 5 + /// 0x79, 0x11, 0x50, 0x00, 0x00, 0x00, 0x00, 0x00, // load mem_end from r1[0x50] to r1 + /// 0x2d, 0x12, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, // if r2 > r1 skip 3 instructions + /// 0x71, 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // load r2 (= *(mem + 5)) into r0 + /// 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 // exit + /// ]; + /// + /// // Instantiate a VM. Note that we provide the start and end offsets for mem pointers. + /// let mut vm = rbpf::EbpfVmFixedMbuff::new(Some(prog), 0x40, 0x50).unwrap(); + /// + /// vm.jit_compile(); + /// ``` + #[cfg(all(not(windows), feature = "std"))] + pub fn jit_compile(&mut self) -> Result<(), Error> { + let prog = match self.parent.prog { + Some(prog) => prog, + None => Err(Error::new( + ErrorKind::Other, + "Error: No program set, call prog_set() to load one", + ))?, + }; + self.parent.jit = Some(jit::JitMemory::new(prog, &self.parent.helpers, true, true)?); + Ok(()) + } + + /// Execute the previously JIT-compiled program, with the given packet data, in a manner very + /// similar to `execute_program()`. + /// + /// If the program is made to be compatible with Linux kernel, it is expected to load the + /// address of the beginning and of the end of the memory area used for packet data from some + /// metadata buffer, which in the case of this VM is handled internally. The offsets at which + /// the addresses should be placed should have be set at the creation of the VM. + /// + /// # Safety + /// + /// **WARNING:** JIT-compiled assembly code is not safe, in particular there is no runtime + /// check for memory access; so if the eBPF program attempts erroneous accesses, this may end + /// very bad (program may segfault). It may be wise to check that the program works with the + /// interpreter before running the JIT-compiled version of it. + /// + /// For this reason the function should be called from within an `unsafe` bloc. + /// + /// # Examples + /// + /// ``` + /// let prog = &[ + /// 0xb7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // mov r0, 0 + /// 0x79, 0x12, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, // load mem from r1[0x40] to r2 + /// 0x07, 0x02, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, // add r2, 5 + /// 0x79, 0x11, 0x50, 0x00, 0x00, 0x00, 0x00, 0x00, // load mem_end from r1[0x50] to r1 + /// 0x2d, 0x12, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, // if r2 > r1 skip 3 instructions + /// 0x71, 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // load r2 (= *(mem + 5)) into r0 + /// 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 // exit + /// ]; + /// let mem = &mut [ + /// 0xaa, 0xbb, 0x11, 0x22, 0xcc, 0xdd + /// ]; + /// + /// // Instantiate a VM. Note that we provide the start and end offsets for mem pointers. + /// let mut vm = rbpf::EbpfVmFixedMbuff::new(Some(prog), 0x40, 0x50).unwrap(); + /// + /// # #[cfg(all(not(windows), feature = "std"))] + /// vm.jit_compile(); + /// + /// // Provide only a reference to the packet data. We do not manage the metadata buffer. + /// # #[cfg(all(not(windows), feature = "std"))] + /// unsafe { + /// let res = vm.execute_program_jit(mem).unwrap(); + /// assert_eq!(res, 0xdd); + /// } + /// ``` + // This struct redefines the `execute_program_jit()` function, in order to pass the offsets + // associated with the fixed mbuff. + #[cfg(all(not(windows), feature = "std"))] + pub unsafe fn execute_program_jit(&mut self, mem: &'a mut [u8]) -> Result { + // If packet data is empty, do not send the address of an empty slice; send a null pointer + // as first argument instead, as this is uBPF's behavior (empty packet should not happen + // in the kernel; anyway the verifier would prevent the use of uninitialized registers). + // See `mul_loop` test. + let mem_ptr = match mem.len() { + 0 => core::ptr::null_mut(), + _ => mem.as_ptr() as *mut u8, + }; + + match &self.parent.jit { + Some(jit) => Ok(jit.get_prog()( + self.mbuff.buffer.as_ptr() as *mut u8, + self.mbuff.buffer.len(), + mem_ptr, + mem.len(), + self.mbuff.data_offset, + self.mbuff.data_end_offset, + )), + None => Err(Error::new( + ErrorKind::Other, + "Error: program has not been JIT-compiled", + )), + } + } + + /// Compile the loaded program using the Cranelift JIT. + /// + /// If using helper functions, be sure to register them into the VM before calling this + /// function. + /// + /// # Examples + /// + /// ``` + /// let prog = &[ + /// 0xb7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // mov r0, 0 + /// 0x79, 0x12, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, // load mem from r1[0x40] to r2 + /// 0x07, 0x02, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, // add r2, 5 + /// 0x79, 0x11, 0x50, 0x00, 0x00, 0x00, 0x00, 0x00, // load mem_end from r1[0x50] to r1 + /// 0x2d, 0x12, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, // if r2 > r1 skip 3 instructions + /// 0x71, 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // load r2 (= *(mem + 5)) into r0 + /// 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 // exit + /// ]; + /// + /// // Instantiate a VM. Note that we provide the start and end offsets for mem pointers. + /// let mut vm = rbpf::EbpfVmFixedMbuff::new(Some(prog), 0x40, 0x50).unwrap(); + /// + /// vm.cranelift_compile(); + /// ``` + #[cfg(feature = "cranelift")] + pub fn cranelift_compile(&mut self) -> Result<(), Error> { + use crate::cranelift::CraneliftCompiler; + + let prog = match self.parent.prog { + Some(prog) => prog, + None => Err(Error::new( + ErrorKind::Other, + "Error: No program set, call prog_set() to load one", + ))?, + }; + + let mut compiler = CraneliftCompiler::new(self.parent.helpers.clone()); + let program = compiler.compile_function(prog)?; + + self.parent.cranelift_prog = Some(program); + Ok(()) + } + + /// Execute the previously compiled program, with the given packet data and metadata + /// buffer, in a manner very similar to `execute_program()`. + /// + /// If the program is made to be compatible with Linux kernel, it is expected to load the + /// address of the beginning and of the end of the memory area used for packet data from some + /// metadata buffer, which in the case of this VM is handled internally. The offsets at which + /// the addresses should be placed should have be set at the creation of the VM. + /// + /// # Examples + /// + /// ``` + /// let prog = &[ + /// 0xb7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // mov r0, 0 + /// 0x79, 0x12, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, // load mem from r1[0x40] to r2 + /// 0x07, 0x02, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, // add r2, 5 + /// 0x79, 0x11, 0x50, 0x00, 0x00, 0x00, 0x00, 0x00, // load mem_end from r1[0x50] to r1 + /// 0x2d, 0x12, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, // if r2 > r1 skip 3 instructions + /// 0x71, 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // load r2 (= *(mem + 5)) into r0 + /// 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 // exit + /// ]; + /// let mem = &mut [ + /// 0xaa, 0xbb, 0x11, 0x22, 0xcc, 0xdd + /// ]; + /// + /// // Instantiate a VM. Note that we provide the start and end offsets for mem pointers. + /// let mut vm = rbpf::EbpfVmFixedMbuff::new(Some(prog), 0x40, 0x50).unwrap(); + /// + /// vm.cranelift_compile(); + /// + /// // Provide only a reference to the packet data. We do not manage the metadata buffer. + /// let res = vm.execute_program_cranelift(mem).unwrap(); + /// assert_eq!(res, 0xdd); + /// ``` + #[cfg(feature = "cranelift")] + pub fn execute_program_cranelift(&mut self, mem: &'a mut [u8]) -> Result { + // If packet data is empty, do not send the address of an empty slice; send a null pointer + // as first argument instead, as this is uBPF's behavior (empty packet should not happen + // in the kernel; anyway the verifier would prevent the use of uninitialized registers). + // See `mul_loop` test. + let mem_ptr = match mem.len() { + 0 => core::ptr::null_mut(), + _ => mem.as_ptr() as *mut u8, + }; + + let l = self.mbuff.buffer.len(); + // Can this ever happen? Probably not, should be ensured at mbuff creation. + if self.mbuff.data_offset + 8 > l || self.mbuff.data_end_offset + 8 > l { + Err(Error::new(ErrorKind::Other, format!("Error: buffer too small ({:?}), cannot use data_offset {:?} and data_end_offset {:?}", + l, self.mbuff.data_offset, self.mbuff.data_end_offset)))?; + } + LittleEndian::write_u64( + &mut self.mbuff.buffer[(self.mbuff.data_offset)..], + mem.as_ptr() as u64, + ); + LittleEndian::write_u64( + &mut self.mbuff.buffer[(self.mbuff.data_end_offset)..], + mem.as_ptr() as u64 + mem.len() as u64, + ); + + match &self.parent.cranelift_prog { + Some(prog) => Ok(prog.execute( + mem_ptr, + mem.len(), + self.mbuff.buffer.as_ptr() as *mut u8, + self.mbuff.buffer.len(), + )), + None => Err(Error::new( + ErrorKind::Other, + "Error: program has not been compiled with cranelift", + )), + } + } +} + +/// A virtual machine to run eBPF program. This kind of VM is used for programs expecting to work +/// directly on the memory area representing packet data. +/// +/// # Examples +/// +/// ``` +/// let prog = &[ +/// 0x71, 0x11, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, // ldxb r1[0x04], r1 +/// 0x07, 0x01, 0x00, 0x00, 0x00, 0x22, 0x00, 0x00, // add r1, 0x22 +/// 0xbf, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // mov r0, r1 +/// 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 // exit +/// ]; +/// let mem = &mut [ +/// 0xaa, 0xbb, 0x11, 0x22, 0xcc, 0xdd +/// ]; +/// +/// // Instantiate a VM. +/// let vm = rbpf::EbpfVmRaw::new(Some(prog)).unwrap(); +/// +/// // Provide only a reference to the packet data. +/// let res = vm.execute_program(mem).unwrap(); +/// assert_eq!(res, 0x22cc); +/// ``` +pub struct EbpfVmRaw<'a> { + parent: EbpfVmMbuff<'a>, +} + +impl<'a> EbpfVmRaw<'a> { + /// Create a new virtual machine instance, and load an eBPF program into that instance. + /// When attempting to load the program, it passes through a simple verifier. + /// + /// # Examples + /// + /// ``` + /// let prog = &[ + /// 0x71, 0x11, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, // ldxb r1[0x04], r1 + /// 0x07, 0x01, 0x00, 0x00, 0x00, 0x22, 0x00, 0x00, // add r1, 0x22 + /// 0xbf, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // mov r0, r1 + /// 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 // exit + /// ]; + /// + /// // Instantiate a VM. + /// let vm = rbpf::EbpfVmRaw::new(Some(prog)).unwrap(); + /// ``` + pub fn new(prog: Option<&'a [u8]>) -> Result, Error> { + let parent = EbpfVmMbuff::new(prog)?; + Ok(EbpfVmRaw { parent }) + } + + /// Load a new eBPF program into the virtual machine instance. + /// + /// # Examples + /// + /// ``` + /// let prog1 = &[ + /// 0xb7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // mov r0, 0 + /// 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 // exit + /// ]; + /// let prog2 = &[ + /// 0x71, 0x11, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, // ldxb r1[0x04], r1 + /// 0x07, 0x01, 0x00, 0x00, 0x00, 0x22, 0x00, 0x00, // add r1, 0x22 + /// 0xbf, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // mov r0, r1 + /// 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 // exit + /// ]; + /// + /// let mem = &mut [ + /// 0xaa, 0xbb, 0x11, 0x22, 0xcc, 0x27, + /// ]; + /// + /// let mut vm = rbpf::EbpfVmRaw::new(Some(prog1)).unwrap(); + /// vm.set_program(prog2); + /// + /// let res = vm.execute_program(mem).unwrap(); + /// assert_eq!(res, 0x22cc); + /// ``` + pub fn set_program(&mut self, prog: &'a [u8]) -> Result<(), Error> { + self.parent.set_program(prog)?; + Ok(()) + } + + /// Set a new verifier function. The function should return an `Error` if the program should be + /// rejected by the virtual machine. If a program has been loaded to the VM already, the + /// verifier is immediately run. + /// + /// # Examples + /// + /// ``` + /// use rbpf::{Error, ErrorKind}; + /// use rbpf::ebpf; + /// + /// // Define a simple verifier function. + /// fn verifier(prog: &[u8]) -> Result<(), Error> { + /// let last_insn = ebpf::get_insn(prog, (prog.len() / ebpf::INSN_SIZE) - 1); + /// if last_insn.opc != ebpf::EXIT { + /// return Err(Error::new(ErrorKind::Other, + /// "[Verifier] Error: program does not end with “EXIT” instruction")); + /// } + /// Ok(()) + /// } + /// + /// let prog1 = &[ + /// 0xb7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // mov r0, 0 + /// 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 // exit + /// ]; + /// + /// // Instantiate a VM. + /// let mut vm = rbpf::EbpfVmMbuff::new(Some(prog1)).unwrap(); + /// // Change the verifier. + /// vm.set_verifier(verifier).unwrap(); + /// ``` + pub fn set_verifier(&mut self, verifier: Verifier) -> Result<(), Error> { + self.parent.set_verifier(verifier) + } + + /// Register a built-in or user-defined helper function in order to use it later from within + /// the eBPF program. The helper is registered into a hashmap, so the `key` can be any `u32`. + /// + /// If using JIT-compiled eBPF programs, be sure to register all helpers before compiling the + /// program. You should be able to change registered helpers after compiling, but not to add + /// new ones (i.e. with new keys). + /// + /// # Examples + /// + /// ``` + /// #[cfg(feature = "std")] { + /// use rbpf::helpers; + /// + /// let prog = &[ + /// 0x79, 0x11, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // ldxdw r1, r1[0x00] + /// 0xb7, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // mov r2, 0 + /// 0xb7, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // mov r3, 0 + /// 0xb7, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // mov r4, 0 + /// 0xb7, 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // mov r5, 0 + /// 0x85, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, // call helper with key 1 + /// 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 // exit + /// ]; + /// + /// let mem = &mut [ + /// 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01 + /// ]; + /// + /// // Instantiate a VM. + /// let mut vm = rbpf::EbpfVmRaw::new(Some(prog)).unwrap(); + /// + /// // Register a helper. This helper will store the result of the square root of r1 into r0. + /// vm.register_helper(1, helpers::sqrti); + /// + /// let res = vm.execute_program(mem).unwrap(); + /// assert_eq!(res, 0x10000000); + /// } + /// ``` + pub fn register_helper( + &mut self, + key: u32, + function: fn(u64, u64, u64, u64, u64) -> u64, + ) -> Result<(), Error> { + self.parent.register_helper(key, function) + } + + /// Register a set of built-in or user-defined helper functions in order to use them later from + /// within the eBPF program. The helpers are registered into a hashmap, so the `key` can be any + /// `u32`. + #[allow(clippy::type_complexity)] + pub fn register_helper_set( + &mut self, + helpers: &HashMap u64>, + ) -> Result<(), Error> { + for (key, function) in helpers { + self.parent.register_helper(*key, *function)?; + } + Ok(()) + } + + /// Execute the program loaded, with the given packet data. + /// + /// # Examples + /// + /// ``` + /// let prog = &[ + /// 0x71, 0x11, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, // ldxb r1[0x04], r1 + /// 0x07, 0x01, 0x00, 0x00, 0x00, 0x22, 0x00, 0x00, // add r1, 0x22 + /// 0xbf, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // mov r0, r1 + /// 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 // exit + /// ]; + /// + /// let mem = &mut [ + /// 0xaa, 0xbb, 0x11, 0x22, 0xcc, 0x27 + /// ]; + /// + /// let mut vm = rbpf::EbpfVmRaw::new(Some(prog)).unwrap(); + /// + /// let res = vm.execute_program(mem).unwrap(); + /// assert_eq!(res, 0x22cc); + /// ``` + pub fn execute_program(&self, mem: &'a mut [u8]) -> Result { + self.parent.execute_program(mem, &[]) + } + + /// JIT-compile the loaded program. No argument required for this. + /// + /// If using helper functions, be sure to register them into the VM before calling this + /// function. + /// + /// # Examples + /// + /// ``` + /// let prog = &[ + /// 0x71, 0x11, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, // ldxb r1[0x04], r1 + /// 0x07, 0x01, 0x00, 0x00, 0x00, 0x22, 0x00, 0x00, // add r1, 0x22 + /// 0xbf, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // mov r0, r1 + /// 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 // exit + /// ]; + /// + /// let mut vm = rbpf::EbpfVmRaw::new(Some(prog)).unwrap(); + /// + /// vm.jit_compile(); + /// ``` + #[cfg(all(not(windows), feature = "std"))] + pub fn jit_compile(&mut self) -> Result<(), Error> { + let prog = match self.parent.prog { + Some(prog) => prog, + None => Err(Error::new( + ErrorKind::Other, + "Error: No program set, call prog_set() to load one", + ))?, + }; + self.parent.jit = Some(jit::JitMemory::new( + prog, + &self.parent.helpers, + false, + false, + )?); + Ok(()) + } + + /// Execute the previously JIT-compiled program, with the given packet data, in a manner very + /// similar to `execute_program()`. + /// + /// # Safety + /// + /// **WARNING:** JIT-compiled assembly code is not safe, in particular there is no runtime + /// check for memory access; so if the eBPF program attempts erroneous accesses, this may end + /// very bad (program may segfault). It may be wise to check that the program works with the + /// interpreter before running the JIT-compiled version of it. + /// + /// For this reason the function should be called from within an `unsafe` bloc. + /// + /// # Examples + /// + /// ``` + /// let prog = &[ + /// 0x71, 0x11, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, // ldxb r1[0x04], r1 + /// 0x07, 0x01, 0x00, 0x00, 0x00, 0x22, 0x00, 0x00, // add r1, 0x22 + /// 0xbf, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // mov r0, r1 + /// 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 // exit + /// ]; + /// + /// let mem = &mut [ + /// 0xaa, 0xbb, 0x11, 0x22, 0xcc, 0x27 + /// ]; + /// + /// let mut vm = rbpf::EbpfVmRaw::new(Some(prog)).unwrap(); + /// + /// # #[cfg(all(not(windows), feature = "std"))] + /// vm.jit_compile(); + /// + /// # #[cfg(all(not(windows), feature = "std"))] + /// unsafe { + /// let res = vm.execute_program_jit(mem).unwrap(); + /// assert_eq!(res, 0x22cc); + /// } + /// ``` + #[cfg(all(not(windows), feature = "std"))] + pub unsafe fn execute_program_jit(&self, mem: &'a mut [u8]) -> Result { + let mut mbuff = vec![]; + self.parent.execute_program_jit(mem, &mut mbuff) + } + + /// Compile the loaded program using the Cranelift JIT. + /// + /// If using helper functions, be sure to register them into the VM before calling this + /// function. + /// + /// # Examples + /// + /// ``` + /// let prog = &[ + /// 0x71, 0x11, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, // ldxb r1[0x04], r1 + /// 0x07, 0x01, 0x00, 0x00, 0x00, 0x22, 0x00, 0x00, // add r1, 0x22 + /// 0xbf, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // mov r0, r1 + /// 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 // exit + /// ]; + /// + /// let mut vm = rbpf::EbpfVmRaw::new(Some(prog)).unwrap(); + /// + /// vm.cranelift_compile(); + /// ``` + #[cfg(feature = "cranelift")] + pub fn cranelift_compile(&mut self) -> Result<(), Error> { + use crate::cranelift::CraneliftCompiler; + + let prog = match self.parent.prog { + Some(prog) => prog, + None => Err(Error::new( + ErrorKind::Other, + "Error: No program set, call prog_set() to load one", + ))?, + }; + + let mut compiler = CraneliftCompiler::new(self.parent.helpers.clone()); + let program = compiler.compile_function(prog)?; + + self.parent.cranelift_prog = Some(program); + Ok(()) + } + + /// Execute the previously compiled program, with the given packet data, in a manner very + /// similar to `execute_program()`. + /// + /// # Examples + /// + /// ``` + /// let prog = &[ + /// 0x71, 0x11, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, // ldxb r1[0x04], r1 + /// 0x07, 0x01, 0x00, 0x00, 0x00, 0x22, 0x00, 0x00, // add r1, 0x22 + /// 0xbf, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // mov r0, r1 + /// 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 // exit + /// ]; + /// + /// let mem = &mut [ + /// 0xaa, 0xbb, 0x11, 0x22, 0xcc, 0x27 + /// ]; + /// + /// let mut vm = rbpf::EbpfVmRaw::new(Some(prog)).unwrap(); + /// + /// vm.cranelift_compile(); + /// + /// let res = vm.execute_program_cranelift(mem).unwrap(); + /// assert_eq!(res, 0x22cc); + /// ``` + #[cfg(feature = "cranelift")] + pub fn execute_program_cranelift(&self, mem: &'a mut [u8]) -> Result { + let mut mbuff = vec![]; + self.parent.execute_program_cranelift(mem, &mut mbuff) + } +} + +/// A virtual machine to run eBPF program. This kind of VM is used for programs that do not work +/// with any memory area—no metadata buffer, no packet data either. +/// +/// # Examples +/// +/// ``` +/// let prog = &[ +/// 0xb7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // mov r0, 0 +/// 0xb7, 0x01, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, // mov r1, 1 +/// 0xb7, 0x02, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, // mov r2, 2 +/// 0xb7, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, // mov r3, 3 +/// 0xb7, 0x04, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, // mov r4, 4 +/// 0xb7, 0x05, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, // mov r5, 5 +/// 0xb7, 0x06, 0x00, 0x00, 0x06, 0x00, 0x00, 0x00, // mov r6, 6 +/// 0xb7, 0x07, 0x00, 0x00, 0x07, 0x00, 0x00, 0x00, // mov r7, 7 +/// 0xb7, 0x08, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, // mov r8, 8 +/// 0x4f, 0x50, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // or r0, r5 +/// 0x47, 0x00, 0x00, 0x00, 0xa0, 0x00, 0x00, 0x00, // or r0, 0xa0 +/// 0x57, 0x00, 0x00, 0x00, 0xa3, 0x00, 0x00, 0x00, // and r0, 0xa3 +/// 0xb7, 0x09, 0x00, 0x00, 0x91, 0x00, 0x00, 0x00, // mov r9, 0x91 +/// 0x5f, 0x90, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // and r0, r9 +/// 0x67, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, // lsh r0, 32 +/// 0x67, 0x00, 0x00, 0x00, 0x16, 0x00, 0x00, 0x00, // lsh r0, 22 +/// 0x6f, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // lsh r0, r8 +/// 0x77, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, // rsh r0, 32 +/// 0x77, 0x00, 0x00, 0x00, 0x13, 0x00, 0x00, 0x00, // rsh r0, 19 +/// 0x7f, 0x70, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // rsh r0, r7 +/// 0xa7, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, // xor r0, 0x03 +/// 0xaf, 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // xor r0, r2 +/// 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 // exit +/// ]; +/// +/// // Instantiate a VM. +/// let vm = rbpf::EbpfVmNoData::new(Some(prog)).unwrap(); +/// +/// // Provide only a reference to the packet data. +/// let res = vm.execute_program().unwrap(); +/// assert_eq!(res, 0x11); +/// ``` +pub struct EbpfVmNoData<'a> { + parent: EbpfVmRaw<'a>, +} + +impl<'a> EbpfVmNoData<'a> { + /// Create a new virtual machine instance, and load an eBPF program into that instance. + /// When attempting to load the program, it passes through a simple verifier. + /// + /// # Examples + /// + /// ``` + /// let prog = &[ + /// 0xb7, 0x00, 0x00, 0x00, 0x11, 0x22, 0x00, 0x00, // mov r0, 0x2211 + /// 0xdc, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, // be16 r0 + /// 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 // exit + /// ]; + /// + /// // Instantiate a VM. + /// let vm = rbpf::EbpfVmNoData::new(Some(prog)); + /// ``` + pub fn new(prog: Option<&'a [u8]>) -> Result, Error> { + let parent = EbpfVmRaw::new(prog)?; + Ok(EbpfVmNoData { parent }) + } + + /// Load a new eBPF program into the virtual machine instance. + /// + /// # Examples + /// + /// ``` + /// let prog1 = &[ + /// 0xb7, 0x00, 0x00, 0x00, 0x11, 0x22, 0x00, 0x00, // mov r0, 0x2211 + /// 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 // exit + /// ]; + /// let prog2 = &[ + /// 0xb7, 0x00, 0x00, 0x00, 0x11, 0x22, 0x00, 0x00, // mov r0, 0x2211 + /// 0xdc, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, // be16 r0 + /// 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 // exit + /// ]; + /// + /// let mut vm = rbpf::EbpfVmNoData::new(Some(prog1)).unwrap(); + /// + /// let res = vm.execute_program().unwrap(); + /// assert_eq!(res, 0x2211); + /// + /// vm.set_program(prog2); + /// + /// let res = vm.execute_program().unwrap(); + /// assert_eq!(res, 0x1122); + /// ``` + pub fn set_program(&mut self, prog: &'a [u8]) -> Result<(), Error> { + self.parent.set_program(prog)?; + Ok(()) + } + + /// Set a new verifier function. The function should return an `Error` if the program should be + /// rejected by the virtual machine. If a program has been loaded to the VM already, the + /// verifier is immediately run. + /// + /// # Examples + /// + /// ``` + /// use rbpf::{Error, ErrorKind}; + /// use rbpf::ebpf; + /// + /// // Define a simple verifier function. + /// fn verifier(prog: &[u8]) -> Result<(), Error> { + /// let last_insn = ebpf::get_insn(prog, (prog.len() / ebpf::INSN_SIZE) - 1); + /// if last_insn.opc != ebpf::EXIT { + /// return Err(Error::new(ErrorKind::Other, + /// "[Verifier] Error: program does not end with “EXIT” instruction")); + /// } + /// Ok(()) + /// } + /// + /// let prog1 = &[ + /// 0xb7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // mov r0, 0 + /// 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 // exit + /// ]; + /// + /// // Instantiate a VM. + /// let mut vm = rbpf::EbpfVmMbuff::new(Some(prog1)).unwrap(); + /// // Change the verifier. + /// vm.set_verifier(verifier).unwrap(); + /// ``` + pub fn set_verifier(&mut self, verifier: Verifier) -> Result<(), Error> { + self.parent.set_verifier(verifier) + } + + /// Register a built-in or user-defined helper function in order to use it later from within + /// the eBPF program. The helper is registered into a hashmap, so the `key` can be any `u32`. + /// + /// If using JIT-compiled eBPF programs, be sure to register all helpers before compiling the + /// program. You should be able to change registered helpers after compiling, but not to add + /// new ones (i.e. with new keys). + /// + /// # Examples + /// + /// ``` + /// #[cfg(feature = "std")] { + /// use rbpf::helpers; + /// + /// let prog = &[ + /// 0xb7, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, // mov r1, 0x010000000 + /// 0xb7, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // mov r2, 0 + /// 0xb7, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // mov r3, 0 + /// 0xb7, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // mov r4, 0 + /// 0xb7, 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // mov r5, 0 + /// 0x85, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, // call helper with key 1 + /// 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 // exit + /// ]; + /// + /// let mut vm = rbpf::EbpfVmNoData::new(Some(prog)).unwrap(); + /// + /// // Register a helper. This helper will store the result of the square root of r1 into r0. + /// vm.register_helper(1, helpers::sqrti).unwrap(); + /// + /// let res = vm.execute_program().unwrap(); + /// assert_eq!(res, 0x1000); + /// } + /// ``` + pub fn register_helper( + &mut self, + key: u32, + function: fn(u64, u64, u64, u64, u64) -> u64, + ) -> Result<(), Error> { + self.parent.register_helper(key, function) + } + + /// JIT-compile the loaded program. No argument required for this. + /// + /// If using helper functions, be sure to register them into the VM before calling this + /// function. + /// + /// # Examples + /// + /// ``` + /// let prog = &[ + /// 0xb7, 0x00, 0x00, 0x00, 0x11, 0x22, 0x00, 0x00, // mov r0, 0x2211 + /// 0xdc, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, // be16 r0 + /// 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 // exit + /// ]; + /// + /// let mut vm = rbpf::EbpfVmNoData::new(Some(prog)).unwrap(); + /// + /// + /// vm.jit_compile(); + /// ``` + #[cfg(all(not(windows), feature = "std"))] + pub fn jit_compile(&mut self) -> Result<(), Error> { + self.parent.jit_compile() + } + + /// Execute the program loaded, without providing pointers to any memory area whatsoever. + /// + /// # Examples + /// + /// ``` + /// let prog = &[ + /// 0xb7, 0x00, 0x00, 0x00, 0x11, 0x22, 0x00, 0x00, // mov r0, 0x2211 + /// 0xdc, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, // be16 r0 + /// 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 // exit + /// ]; + /// + /// let vm = rbpf::EbpfVmNoData::new(Some(prog)).unwrap(); + /// + /// // For this kind of VM, the `execute_program()` function needs no argument. + /// let res = vm.execute_program().unwrap(); + /// assert_eq!(res, 0x1122); + /// ``` + pub fn execute_program(&self) -> Result { + self.parent.execute_program(&mut []) + } + + /// Execute the previously JIT-compiled program, without providing pointers to any memory area + /// whatsoever, in a manner very similar to `execute_program()`. + /// + /// # Safety + /// + /// **WARNING:** JIT-compiled assembly code is not safe, in particular there is no runtime + /// check for memory access; so if the eBPF program attempts erroneous accesses, this may end + /// very bad (program may segfault). It may be wise to check that the program works with the + /// interpreter before running the JIT-compiled version of it. + /// + /// For this reason the function should be called from within an `unsafe` bloc. + /// + /// # Examples + /// + /// ``` + /// let prog = &[ + /// 0xb7, 0x00, 0x00, 0x00, 0x11, 0x22, 0x00, 0x00, // mov r0, 0x2211 + /// 0xdc, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, // be16 r0 + /// 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 // exit + /// ]; + /// + /// let mut vm = rbpf::EbpfVmNoData::new(Some(prog)).unwrap(); + /// + /// # #[cfg(all(not(windows), feature = "std"))] + /// vm.jit_compile(); + /// + /// # #[cfg(all(not(windows), feature = "std"))] + /// unsafe { + /// let res = vm.execute_program_jit().unwrap(); + /// assert_eq!(res, 0x1122); + /// } + /// ``` + #[cfg(all(not(windows), feature = "std"))] + pub unsafe fn execute_program_jit(&self) -> Result { + self.parent.execute_program_jit(&mut []) + } + + /// Compile the loaded program using the Cranelift JIT. + /// + /// If using helper functions, be sure to register them into the VM before calling this + /// function. + /// + /// # Examples + /// + /// ``` + /// let prog = &[ + /// 0xb7, 0x00, 0x00, 0x00, 0x11, 0x22, 0x00, 0x00, // mov r0, 0x2211 + /// 0xdc, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, // be16 r0 + /// 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 // exit + /// ]; + /// + /// let mut vm = rbpf::EbpfVmNoData::new(Some(prog)).unwrap(); + /// + /// + /// vm.cranelift_compile(); + /// ``` + #[cfg(feature = "cranelift")] + pub fn cranelift_compile(&mut self) -> Result<(), Error> { + self.parent.cranelift_compile() + } + + /// Execute the previously JIT-compiled program, without providing pointers to any memory area + /// whatsoever, in a manner very similar to `execute_program()`. + /// + /// # Examples + /// + /// ``` + /// let prog = &[ + /// 0xb7, 0x00, 0x00, 0x00, 0x11, 0x22, 0x00, 0x00, // mov r0, 0x2211 + /// 0xdc, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, // be16 r0 + /// 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 // exit + /// ]; + /// + /// let mut vm = rbpf::EbpfVmNoData::new(Some(prog)).unwrap(); + /// + /// vm.cranelift_compile(); + /// + /// let res = vm.execute_program_cranelift().unwrap(); + /// assert_eq!(res, 0x1122); + /// ``` + #[cfg(feature = "cranelift")] + pub fn execute_program_cranelift(&self) -> Result { + self.parent.execute_program_cranelift(&mut []) + } +} + +/// EbpfVm with Owned data +pub struct EbpfVmRawOwned { + parent: EbpfVmRaw<'static>, + data_len: usize, + data_cap: usize, +} + +impl EbpfVmRawOwned { + /// Create a new virtual machine instance, and load an eBPF program into that instance. + /// When attempting to load the program, it passes through a simple verifier. + pub fn new(prog: Option>) -> Result { + let (prog, data_len, data_cap) = match prog { + Some(prog) => { + let data_len = prog.len(); + let data_cap = prog.capacity(); + let slice = prog.leak(); + let slice = unsafe { core::slice::from_raw_parts(slice.as_ptr(), data_len) }; + (Some(slice), data_len, data_cap) + } + None => (None, 0, 0), + }; + let parent = EbpfVmRaw::new(prog)?; + Ok(Self { + parent, + data_len, + data_cap, + }) + } + /// Load a new eBPF program into the virtual machine instance + pub fn set_program(&mut self, prog: Vec) -> Result<(), Error> { + self.data_len = prog.len(); + self.data_cap = prog.capacity(); + let slice = prog.leak(); + self.parent.set_program(slice)?; + Ok(()) + } + + /// Set a new verifier function. The function should return an Error if the program should be rejected by the virtual machine. + /// If a program has been loaded to the VM already, the verifier is immediately run. + pub fn set_verifier(&mut self, verifier: Verifier) -> Result<(), Error> { + self.parent.set_verifier(verifier) + } + + /// Register a built-in or user-defined helper function in order to use it later from within the eBPF program. + /// The helper is registered into a hashmap, so the key can be any u32. + /// If using JIT-compiled eBPF programs, be sure to register all helpers before compiling the program. + /// You should be able to change registered helpers after compiling, but not to add new ones (i. e. with new keys). + pub fn register_helper( + &mut self, + key: u32, + function: fn(u64, u64, u64, u64, u64) -> u64, + ) -> Result<(), Error> { + self.parent.register_helper(key, function) + } + + /// Register a set of built-in or user-defined helper functions in order to use them later from + /// within the eBPF program. The helpers are registered into a hashmap, so the `key` can be any + /// `u32`. + #[allow(clippy::type_complexity)] + pub fn register_helper_set( + &mut self, + helpers: &HashMap u64>, + ) -> Result<(), Error> { + for (key, function) in helpers { + self.parent.register_helper(*key, *function)?; + } + Ok(()) + } + + /// Execute the previously JIT-compiled program, with the given packet data, in a manner very similar to execute_program(). + /// + /// Safety + /// + /// **WARNING:** JIT-compiled assembly code is not safe, in particular there is no runtime check for memory access; + /// so if the eBPF program attempts erroneous accesses, this may end very bad (program may segfault). + /// It may be wise to check that the program works with the interpreter before running the JIT-compiled version of it. + /// + /// For this reason the function should be called from within an unsafe bloc. + pub fn execute_program(&self, mem: &mut [u8]) -> Result { + self.parent.execute_program(mem) + } +} + +impl Drop for EbpfVmRawOwned { + fn drop(&mut self) { + match self.parent.parent.prog { + Some(prog) => unsafe { + let ptr = prog.as_ptr(); + let _prog = Vec::from_raw_parts(ptr as *mut u8, self.data_len, self.data_cap); + }, + None => {} + }; + } +} diff --git a/kernel/crates/rbpf/src/no_std_error.rs b/kernel/crates/rbpf/src/no_std_error.rs new file mode 100644 index 000000000..f502eecff --- /dev/null +++ b/kernel/crates/rbpf/src/no_std_error.rs @@ -0,0 +1,41 @@ +// SPDX-License-Identifier: (Apache-2.0 OR MIT) + +//! This module provides a simple implementation of the Error struct that is +//! used as a drop-in replacement for `std::io::Error` when using `rbpf` in `no_std`. + +use alloc::string::String; + +/// Implementation of Error for no_std applications. +/// Ensures that the existing code can use it with the same interface +/// as the Error from std::io::Error. +#[derive(Debug)] +pub struct Error { + #[allow(dead_code)] + kind: ErrorKind, + #[allow(dead_code)] + error: String, +} + +impl Error { + /// New function exposing the same signature as `std::io::Error::new`. + #[allow(dead_code)] + pub fn new>(kind: ErrorKind, error: S) -> Error { + Error { + kind, + error: error.into(), + } + } +} + +/// The current version of `rbpf` only uses the [`Other`](ErrorKind::Other) variant +/// from the [std::io::ErrorKind] enum. If a dependency on other variants were +/// introduced in the future, this enum needs to be updated accordingly to maintain +/// compatibility with the real `ErrorKind`. The reason all available variants +/// aren't included in the first place is that [std::io::ErrorKind] exposes +/// 40 variants, and not all of them are meaningful under `no_std`. +#[derive(Debug)] +pub enum ErrorKind { + /// The no_std code only uses this variant. + #[allow(dead_code)] + Other, +} diff --git a/kernel/crates/rbpf/src/stack.rs b/kernel/crates/rbpf/src/stack.rs new file mode 100644 index 000000000..be1732638 --- /dev/null +++ b/kernel/crates/rbpf/src/stack.rs @@ -0,0 +1,75 @@ +use crate::{ebpf::STACK_SIZE, vec, Vec}; + +pub struct StackFrame { + return_address: u16, + saved_registers: [u64; 4], + sp: u16, + frame: Vec, +} + +impl StackFrame { + /// Create a new stack frame + /// + /// The stack frame is created with a capacity of `STACK_SIZE` == 512 bytes + pub fn new() -> Self { + Self { + sp: 0, + return_address: 0, + saved_registers: [0; 4], + frame: vec![0; STACK_SIZE], + } + } + + /// Create a new stack frame with a given capacity + #[allow(unused)] + pub fn with_capacity(capacity: usize) -> Self { + Self { + sp: 0, + return_address: 0, + saved_registers: [0; 4], + frame: vec![0; capacity], + } + } + + /// The capacity of the stack frame + pub fn len(&self) -> usize { + self.frame.len() + } + + pub fn as_ptr(&self) -> *const u8 { + self.frame.as_ptr() + } + + pub fn as_slice(&self) -> &[u8] { + self.frame.as_slice() + } + /// Save the callee-saved registers + pub fn save_registers(&mut self, regs: &[u64]) { + self.saved_registers.copy_from_slice(regs); + } + + /// Get the callee-saved registers + pub fn get_registers(&self) -> [u64; 4] { + self.saved_registers + } + + /// Save the return address + pub fn save_return_address(&mut self, address: u16) { + self.return_address = address; + } + + /// Get the return address + pub fn get_return_address(&self) -> u16 { + self.return_address + } + + /// Save the stack pointer + pub fn save_sp(&mut self, sp: u16) { + self.sp = sp; + } + + /// Get the stack pointer + pub fn get_sp(&self) -> u16 { + self.sp + } +} diff --git a/kernel/crates/rbpf/src/verifier.rs b/kernel/crates/rbpf/src/verifier.rs new file mode 100644 index 000000000..4c9cf2507 --- /dev/null +++ b/kernel/crates/rbpf/src/verifier.rs @@ -0,0 +1,386 @@ +// SPDX-License-Identifier: (Apache-2.0 OR MIT) +// Derived from uBPF +// Copyright 2015 Big Switch Networks, Inc +// (uBPF: safety checks, originally in C) +// Copyright 2016 6WIND S.A. +// (Translation to Rust) + +// This “verifier” performs simple checks when the eBPF program is loaded into the VM (before it is +// interpreted or JIT-compiled). It has nothing to do with the much more elaborated verifier inside +// Linux kernel. There is no verification regarding the program flow control (should be a Direct +// Acyclic Graph) or the consistency for registers usage (the verifier of the kernel assigns types +// to the registers and is much stricter). +// +// On the other hand, rbpf is not expected to run in kernel space. +// +// Improving the verifier would be nice, but this is not trivial (and Linux kernel is under GPL +// license, so we cannot copy it). +// +// Contrary to the verifier of the Linux kernel, this one does not modify the bytecode at all. + +use alloc::format; + +use crate::{ebpf, Error, ErrorKind}; + +fn reject>(msg: S) -> Result<(), Error> { + let full_msg = format!("[Verifier] Error: {}", msg.as_ref()); + Err(Error::new(ErrorKind::Other, full_msg)) +} + +fn check_prog_len(prog: &[u8]) -> Result<(), Error> { + if prog.len() % ebpf::INSN_SIZE != 0 { + reject(format!( + "eBPF program length must be a multiple of {:?} octets", + ebpf::INSN_SIZE + ))?; + } + if prog.len() > ebpf::PROG_MAX_SIZE { + reject(format!( + "eBPF program length limited to {:?}, here {:?}", + ebpf::PROG_MAX_INSNS, + prog.len() / ebpf::INSN_SIZE + ))?; + } + + if prog.is_empty() { + reject("no program set, call set_program() to load one")?; + } + let last_opc = ebpf::get_insn(prog, (prog.len() / ebpf::INSN_SIZE) - 1).opc; + if last_opc & ebpf::BPF_CLS_MASK != ebpf::BPF_JMP { + reject("program does not end with “EXIT” instruction")?; + } + + Ok(()) +} + +fn check_imm_endian(insn: &ebpf::Insn, insn_ptr: usize) -> Result<(), Error> { + match insn.imm { + 16 | 32 | 64 => Ok(()), + _ => reject(format!( + "unsupported argument for LE/BE (insn #{insn_ptr:?})" + )), + } +} + +fn check_load_dw(prog: &[u8], insn_ptr: usize) -> Result<(), Error> { + // We know we can reach next insn since we enforce an EXIT insn at the end of program, while + // this function should be called only for LD_DW insn, that cannot be last in program. + let next_insn = ebpf::get_insn(prog, insn_ptr + 1); + if next_insn.opc != 0 { + reject(format!("incomplete LD_DW instruction (insn #{insn_ptr:?})"))?; + } + + Ok(()) +} + +fn check_jmp_offset(prog: &[u8], insn_ptr: usize) -> Result<(), Error> { + let insn = ebpf::get_insn(prog, insn_ptr); + if insn.off == -1 { + reject(format!("infinite loop (insn #{insn_ptr:?})"))?; + } + + let dst_insn_ptr = insn_ptr as isize + 1 + insn.off as isize; + if dst_insn_ptr < 0 || dst_insn_ptr as usize >= (prog.len() / ebpf::INSN_SIZE) { + reject(format!( + "jump out of code to #{dst_insn_ptr:?} (insn #{insn_ptr:?})" + ))?; + } + + let dst_insn = ebpf::get_insn(prog, dst_insn_ptr as usize); + if dst_insn.opc == 0 { + reject(format!( + "jump to middle of LD_DW at #{dst_insn_ptr:?} (insn #{insn_ptr:?})" + ))?; + } + + Ok(()) +} + +fn check_registers(insn: &ebpf::Insn, store: bool, insn_ptr: usize) -> Result<(), Error> { + if insn.src > 10 { + reject(format!("invalid source register (insn #{insn_ptr:?})"))?; + } + + match (insn.dst, store) { + (0..=9, _) | (10, true) => Ok(()), + (10, false) => reject(format!( + "cannot write into register r10 (insn #{insn_ptr:?})" + )), + (_, _) => reject(format!("invalid destination register (insn #{insn_ptr:?})")), + } +} + +pub fn check(prog: &[u8]) -> Result<(), Error> { + check_prog_len(prog)?; + + let mut insn_ptr: usize = 0; + while insn_ptr * ebpf::INSN_SIZE < prog.len() { + let insn = ebpf::get_insn(prog, insn_ptr); + let mut store = false; + + match insn.opc { + // BPF_LD class + ebpf::LD_ABS_B => {} + ebpf::LD_ABS_H => {} + ebpf::LD_ABS_W => {} + ebpf::LD_ABS_DW => {} + ebpf::LD_IND_B => {} + ebpf::LD_IND_H => {} + ebpf::LD_IND_W => {} + ebpf::LD_IND_DW => {} + + ebpf::LD_DW_IMM => { + store = true; + check_load_dw(prog, insn_ptr)?; + insn_ptr += 1; + } + + // BPF_LDX class + ebpf::LD_B_REG => {} + ebpf::LD_H_REG => {} + ebpf::LD_W_REG => {} + ebpf::LD_DW_REG => {} + + // BPF_ST class + ebpf::ST_B_IMM => store = true, + ebpf::ST_H_IMM => store = true, + ebpf::ST_W_IMM => store = true, + ebpf::ST_DW_IMM => store = true, + + // BPF_STX class + ebpf::ST_B_REG => store = true, + ebpf::ST_H_REG => store = true, + ebpf::ST_W_REG => store = true, + ebpf::ST_DW_REG => store = true, + ebpf::ST_W_XADD => { + unimplemented!(); + } + ebpf::ST_DW_XADD => { + unimplemented!(); + } + + // BPF_ALU class + ebpf::ADD32_IMM => {} + ebpf::ADD32_REG => {} + ebpf::SUB32_IMM => {} + ebpf::SUB32_REG => {} + ebpf::MUL32_IMM => {} + ebpf::MUL32_REG => {} + ebpf::DIV32_IMM => {} + ebpf::DIV32_REG => {} + ebpf::OR32_IMM => {} + ebpf::OR32_REG => {} + ebpf::AND32_IMM => {} + ebpf::AND32_REG => {} + ebpf::LSH32_IMM => {} + ebpf::LSH32_REG => {} + ebpf::RSH32_IMM => {} + ebpf::RSH32_REG => {} + ebpf::NEG32 => {} + ebpf::MOD32_IMM => {} + ebpf::MOD32_REG => {} + ebpf::XOR32_IMM => {} + ebpf::XOR32_REG => {} + ebpf::MOV32_IMM => {} + ebpf::MOV32_REG => {} + ebpf::ARSH32_IMM => {} + ebpf::ARSH32_REG => {} + ebpf::LE => { + check_imm_endian(&insn, insn_ptr)?; + } + ebpf::BE => { + check_imm_endian(&insn, insn_ptr)?; + } + + // BPF_ALU64 class + ebpf::ADD64_IMM => {} + ebpf::ADD64_REG => {} + ebpf::SUB64_IMM => {} + ebpf::SUB64_REG => {} + ebpf::MUL64_IMM => {} + ebpf::MUL64_REG => {} + ebpf::DIV64_IMM => {} + ebpf::DIV64_REG => {} + ebpf::OR64_IMM => {} + ebpf::OR64_REG => {} + ebpf::AND64_IMM => {} + ebpf::AND64_REG => {} + ebpf::LSH64_IMM => {} + ebpf::LSH64_REG => {} + ebpf::RSH64_IMM => {} + ebpf::RSH64_REG => {} + ebpf::NEG64 => {} + ebpf::MOD64_IMM => {} + ebpf::MOD64_REG => {} + ebpf::XOR64_IMM => {} + ebpf::XOR64_REG => {} + ebpf::MOV64_IMM => {} + ebpf::MOV64_REG => {} + ebpf::ARSH64_IMM => {} + ebpf::ARSH64_REG => {} + + // BPF_JMP class + ebpf::JA => { + check_jmp_offset(prog, insn_ptr)?; + } + ebpf::JEQ_IMM => { + check_jmp_offset(prog, insn_ptr)?; + } + ebpf::JEQ_REG => { + check_jmp_offset(prog, insn_ptr)?; + } + ebpf::JGT_IMM => { + check_jmp_offset(prog, insn_ptr)?; + } + ebpf::JGT_REG => { + check_jmp_offset(prog, insn_ptr)?; + } + ebpf::JGE_IMM => { + check_jmp_offset(prog, insn_ptr)?; + } + ebpf::JGE_REG => { + check_jmp_offset(prog, insn_ptr)?; + } + ebpf::JLT_IMM => { + check_jmp_offset(prog, insn_ptr)?; + } + ebpf::JLT_REG => { + check_jmp_offset(prog, insn_ptr)?; + } + ebpf::JLE_IMM => { + check_jmp_offset(prog, insn_ptr)?; + } + ebpf::JLE_REG => { + check_jmp_offset(prog, insn_ptr)?; + } + ebpf::JSET_IMM => { + check_jmp_offset(prog, insn_ptr)?; + } + ebpf::JSET_REG => { + check_jmp_offset(prog, insn_ptr)?; + } + ebpf::JNE_IMM => { + check_jmp_offset(prog, insn_ptr)?; + } + ebpf::JNE_REG => { + check_jmp_offset(prog, insn_ptr)?; + } + ebpf::JSGT_IMM => { + check_jmp_offset(prog, insn_ptr)?; + } + ebpf::JSGT_REG => { + check_jmp_offset(prog, insn_ptr)?; + } + ebpf::JSGE_IMM => { + check_jmp_offset(prog, insn_ptr)?; + } + ebpf::JSGE_REG => { + check_jmp_offset(prog, insn_ptr)?; + } + ebpf::JSLT_IMM => { + check_jmp_offset(prog, insn_ptr)?; + } + ebpf::JSLT_REG => { + check_jmp_offset(prog, insn_ptr)?; + } + ebpf::JSLE_IMM => { + check_jmp_offset(prog, insn_ptr)?; + } + ebpf::JSLE_REG => { + check_jmp_offset(prog, insn_ptr)?; + } + + // BPF_JMP32 class + ebpf::JEQ_IMM32 => { + check_jmp_offset(prog, insn_ptr)?; + } + ebpf::JEQ_REG32 => { + check_jmp_offset(prog, insn_ptr)?; + } + ebpf::JGT_IMM32 => { + check_jmp_offset(prog, insn_ptr)?; + } + ebpf::JGT_REG32 => { + check_jmp_offset(prog, insn_ptr)?; + } + ebpf::JGE_IMM32 => { + check_jmp_offset(prog, insn_ptr)?; + } + ebpf::JGE_REG32 => { + check_jmp_offset(prog, insn_ptr)?; + } + ebpf::JLT_IMM32 => { + check_jmp_offset(prog, insn_ptr)?; + } + ebpf::JLT_REG32 => { + check_jmp_offset(prog, insn_ptr)?; + } + ebpf::JLE_IMM32 => { + check_jmp_offset(prog, insn_ptr)?; + } + ebpf::JLE_REG32 => { + check_jmp_offset(prog, insn_ptr)?; + } + ebpf::JSET_IMM32 => { + check_jmp_offset(prog, insn_ptr)?; + } + ebpf::JSET_REG32 => { + check_jmp_offset(prog, insn_ptr)?; + } + ebpf::JNE_IMM32 => { + check_jmp_offset(prog, insn_ptr)?; + } + ebpf::JNE_REG32 => { + check_jmp_offset(prog, insn_ptr)?; + } + ebpf::JSGT_IMM32 => { + check_jmp_offset(prog, insn_ptr)?; + } + ebpf::JSGT_REG32 => { + check_jmp_offset(prog, insn_ptr)?; + } + ebpf::JSGE_IMM32 => { + check_jmp_offset(prog, insn_ptr)?; + } + ebpf::JSGE_REG32 => { + check_jmp_offset(prog, insn_ptr)?; + } + ebpf::JSLT_IMM32 => { + check_jmp_offset(prog, insn_ptr)?; + } + ebpf::JSLT_REG32 => { + check_jmp_offset(prog, insn_ptr)?; + } + ebpf::JSLE_IMM32 => { + check_jmp_offset(prog, insn_ptr)?; + } + ebpf::JSLE_REG32 => { + check_jmp_offset(prog, insn_ptr)?; + } + + ebpf::CALL => {} + ebpf::TAIL_CALL => { + unimplemented!() + } + ebpf::EXIT => {} + + _ => { + reject(format!( + "unknown eBPF opcode {:#2x} (insn #{insn_ptr:?})", + insn.opc + ))?; + } + } + + check_registers(&insn, store, insn_ptr)?; + + insn_ptr += 1; + } + + // insn_ptr should now be equal to number of instructions. + if insn_ptr != prog.len() / ebpf::INSN_SIZE { + reject(format!("jumped out of code to #{insn_ptr:?}"))?; + } + + Ok(()) +} diff --git a/kernel/crates/rbpf/tests/assembler.rs b/kernel/crates/rbpf/tests/assembler.rs new file mode 100644 index 000000000..0640772ca --- /dev/null +++ b/kernel/crates/rbpf/tests/assembler.rs @@ -0,0 +1,655 @@ +// SPDX-License-Identifier: (Apache-2.0 OR MIT) +// Copyright 2017 Rich Lane + +#![allow(clippy::unreadable_literal)] + +extern crate rbpf; +mod common; + +use common::{TCP_SACK_ASM, TCP_SACK_BIN}; +use rbpf::{assembler::assemble, ebpf}; + +fn asm(src: &str) -> Result, String> { + Ok(ebpf::to_insn_vec(&(assemble(src))?)) +} + +fn insn(opc: u8, dst: u8, src: u8, off: i16, imm: i32) -> ebpf::Insn { + ebpf::Insn { + opc, + dst, + src, + off, + imm, + } +} + +#[test] +fn test_empty() { + assert_eq!(asm(""), Ok(vec![])); +} + +// Example for InstructionType::NoOperand. +#[test] +fn test_exit() { + assert_eq!(asm("exit"), Ok(vec![insn(ebpf::EXIT, 0, 0, 0, 0)])); +} + +// Example for InstructionType::AluBinary. +#[test] +fn test_add64() { + assert_eq!( + asm("add64 r1, r3"), + Ok(vec![insn(ebpf::ADD64_REG, 1, 3, 0, 0)]) + ); + assert_eq!( + asm("add64 r1, 5"), + Ok(vec![insn(ebpf::ADD64_IMM, 1, 0, 0, 5)]) + ); +} + +// Example for InstructionType::AluUnary. +#[test] +fn test_neg64() { + assert_eq!(asm("neg64 r1"), Ok(vec![insn(ebpf::NEG64, 1, 0, 0, 0)])); +} + +// Example for InstructionType::LoadReg. +#[test] +fn test_ldxw() { + assert_eq!( + asm("ldxw r1, [r2+5]"), + Ok(vec![insn(ebpf::LD_W_REG, 1, 2, 5, 0)]) + ); +} + +// Example for InstructionType::StoreImm. +#[test] +fn test_stw() { + assert_eq!( + asm("stw [r2+5], 7"), + Ok(vec![insn(ebpf::ST_W_IMM, 2, 0, 5, 7)]) + ); +} + +// Example for InstructionType::StoreReg. +#[test] +fn test_stxw() { + assert_eq!( + asm("stxw [r2+5], r8"), + Ok(vec![insn(ebpf::ST_W_REG, 2, 8, 5, 0)]) + ); +} + +// Example for InstructionType::JumpUnconditional. +#[test] +fn test_ja() { + assert_eq!(asm("ja +8"), Ok(vec![insn(ebpf::JA, 0, 0, 8, 0)])); + assert_eq!(asm("ja -3"), Ok(vec![insn(ebpf::JA, 0, 0, -3, 0)])); +} + +// Example for InstructionType::JumpConditional. +#[test] +fn test_jeq() { + assert_eq!( + asm("jeq r1, 4, +8"), + Ok(vec![insn(ebpf::JEQ_IMM, 1, 0, 8, 4)]) + ); + assert_eq!( + asm("jeq r1, r3, +8"), + Ok(vec![insn(ebpf::JEQ_REG, 1, 3, 8, 0)]) + ); +} + +// Example for InstructionType::Call. +#[test] +fn test_call() { + assert_eq!(asm("call 300"), Ok(vec![insn(ebpf::CALL, 0, 0, 0, 300)])); +} + +// Example for InstructionType::Endian. +#[test] +fn test_be32() { + assert_eq!(asm("be32 r1"), Ok(vec![insn(ebpf::BE, 1, 0, 0, 32)])); +} + +// Example for InstructionType::LoadImm. +#[test] +fn test_lddw() { + assert_eq!( + asm("lddw r1, 0x1234abcd5678eeff"), + Ok(vec![ + insn(ebpf::LD_DW_IMM, 1, 0, 0, 0x5678eeff), + insn(0, 0, 0, 0, 0x1234abcd) + ]) + ); + assert_eq!( + asm("lddw r1, 0xff11ee22dd33cc44"), + Ok(vec![ + insn(ebpf::LD_DW_IMM, 1, 0, 0, 0xdd33cc44u32 as i32), + insn(0, 0, 0, 0, 0xff11ee22u32 as i32) + ]) + ); +} + +// Example for InstructionType::LoadAbs. +#[test] +fn test_ldabsw() { + assert_eq!(asm("ldabsw 1"), Ok(vec![insn(ebpf::LD_ABS_W, 0, 0, 0, 1)])); +} + +// Example for InstructionType::LoadInd. +#[test] +fn test_ldindw() { + assert_eq!( + asm("ldindw r1, 2"), + Ok(vec![insn(ebpf::LD_IND_W, 0, 1, 0, 2)]) + ); +} + +// Example for InstructionType::LoadReg. +#[test] +fn test_ldxdw() { + assert_eq!( + asm("ldxdw r1, [r2+3]"), + Ok(vec![insn(ebpf::LD_DW_REG, 1, 2, 3, 0)]) + ); +} + +// Example for InstructionType::StoreImm. +#[test] +fn test_sth() { + assert_eq!( + asm("sth [r1+2], 3"), + Ok(vec![insn(ebpf::ST_H_IMM, 1, 0, 2, 3)]) + ); +} + +// Example for InstructionType::StoreReg. +#[test] +fn test_stxh() { + assert_eq!( + asm("stxh [r1+2], r3"), + Ok(vec![insn(ebpf::ST_H_REG, 1, 3, 2, 0)]) + ); +} + +// Test all supported AluBinary mnemonics. +#[test] +fn test_alu_binary() { + assert_eq!( + asm("add r1, r2 + sub r1, r2 + mul r1, r2 + div r1, r2 + or r1, r2 + and r1, r2 + lsh r1, r2 + rsh r1, r2 + mod r1, r2 + xor r1, r2 + mov r1, r2 + arsh r1, r2"), + Ok(vec![ + insn(ebpf::ADD64_REG, 1, 2, 0, 0), + insn(ebpf::SUB64_REG, 1, 2, 0, 0), + insn(ebpf::MUL64_REG, 1, 2, 0, 0), + insn(ebpf::DIV64_REG, 1, 2, 0, 0), + insn(ebpf::OR64_REG, 1, 2, 0, 0), + insn(ebpf::AND64_REG, 1, 2, 0, 0), + insn(ebpf::LSH64_REG, 1, 2, 0, 0), + insn(ebpf::RSH64_REG, 1, 2, 0, 0), + insn(ebpf::MOD64_REG, 1, 2, 0, 0), + insn(ebpf::XOR64_REG, 1, 2, 0, 0), + insn(ebpf::MOV64_REG, 1, 2, 0, 0), + insn(ebpf::ARSH64_REG, 1, 2, 0, 0) + ]) + ); + + assert_eq!( + asm("add r1, 2 + sub r1, 2 + mul r1, 2 + div r1, 2 + or r1, 2 + and r1, 2 + lsh r1, 2 + rsh r1, 2 + mod r1, 2 + xor r1, 2 + mov r1, 2 + arsh r1, 2"), + Ok(vec![ + insn(ebpf::ADD64_IMM, 1, 0, 0, 2), + insn(ebpf::SUB64_IMM, 1, 0, 0, 2), + insn(ebpf::MUL64_IMM, 1, 0, 0, 2), + insn(ebpf::DIV64_IMM, 1, 0, 0, 2), + insn(ebpf::OR64_IMM, 1, 0, 0, 2), + insn(ebpf::AND64_IMM, 1, 0, 0, 2), + insn(ebpf::LSH64_IMM, 1, 0, 0, 2), + insn(ebpf::RSH64_IMM, 1, 0, 0, 2), + insn(ebpf::MOD64_IMM, 1, 0, 0, 2), + insn(ebpf::XOR64_IMM, 1, 0, 0, 2), + insn(ebpf::MOV64_IMM, 1, 0, 0, 2), + insn(ebpf::ARSH64_IMM, 1, 0, 0, 2) + ]) + ); + + assert_eq!( + asm("add64 r1, r2 + sub64 r1, r2 + mul64 r1, r2 + div64 r1, r2 + or64 r1, r2 + and64 r1, r2 + lsh64 r1, r2 + rsh64 r1, r2 + mod64 r1, r2 + xor64 r1, r2 + mov64 r1, r2 + arsh64 r1, r2"), + Ok(vec![ + insn(ebpf::ADD64_REG, 1, 2, 0, 0), + insn(ebpf::SUB64_REG, 1, 2, 0, 0), + insn(ebpf::MUL64_REG, 1, 2, 0, 0), + insn(ebpf::DIV64_REG, 1, 2, 0, 0), + insn(ebpf::OR64_REG, 1, 2, 0, 0), + insn(ebpf::AND64_REG, 1, 2, 0, 0), + insn(ebpf::LSH64_REG, 1, 2, 0, 0), + insn(ebpf::RSH64_REG, 1, 2, 0, 0), + insn(ebpf::MOD64_REG, 1, 2, 0, 0), + insn(ebpf::XOR64_REG, 1, 2, 0, 0), + insn(ebpf::MOV64_REG, 1, 2, 0, 0), + insn(ebpf::ARSH64_REG, 1, 2, 0, 0) + ]) + ); + + assert_eq!( + asm("add64 r1, 2 + sub64 r1, 2 + mul64 r1, 2 + div64 r1, 2 + or64 r1, 2 + and64 r1, 2 + lsh64 r1, 2 + rsh64 r1, 2 + mod64 r1, 2 + xor64 r1, 2 + mov64 r1, 2 + arsh64 r1, 2"), + Ok(vec![ + insn(ebpf::ADD64_IMM, 1, 0, 0, 2), + insn(ebpf::SUB64_IMM, 1, 0, 0, 2), + insn(ebpf::MUL64_IMM, 1, 0, 0, 2), + insn(ebpf::DIV64_IMM, 1, 0, 0, 2), + insn(ebpf::OR64_IMM, 1, 0, 0, 2), + insn(ebpf::AND64_IMM, 1, 0, 0, 2), + insn(ebpf::LSH64_IMM, 1, 0, 0, 2), + insn(ebpf::RSH64_IMM, 1, 0, 0, 2), + insn(ebpf::MOD64_IMM, 1, 0, 0, 2), + insn(ebpf::XOR64_IMM, 1, 0, 0, 2), + insn(ebpf::MOV64_IMM, 1, 0, 0, 2), + insn(ebpf::ARSH64_IMM, 1, 0, 0, 2) + ]) + ); + + assert_eq!( + asm("add32 r1, r2 + sub32 r1, r2 + mul32 r1, r2 + div32 r1, r2 + or32 r1, r2 + and32 r1, r2 + lsh32 r1, r2 + rsh32 r1, r2 + mod32 r1, r2 + xor32 r1, r2 + mov32 r1, r2 + arsh32 r1, r2"), + Ok(vec![ + insn(ebpf::ADD32_REG, 1, 2, 0, 0), + insn(ebpf::SUB32_REG, 1, 2, 0, 0), + insn(ebpf::MUL32_REG, 1, 2, 0, 0), + insn(ebpf::DIV32_REG, 1, 2, 0, 0), + insn(ebpf::OR32_REG, 1, 2, 0, 0), + insn(ebpf::AND32_REG, 1, 2, 0, 0), + insn(ebpf::LSH32_REG, 1, 2, 0, 0), + insn(ebpf::RSH32_REG, 1, 2, 0, 0), + insn(ebpf::MOD32_REG, 1, 2, 0, 0), + insn(ebpf::XOR32_REG, 1, 2, 0, 0), + insn(ebpf::MOV32_REG, 1, 2, 0, 0), + insn(ebpf::ARSH32_REG, 1, 2, 0, 0) + ]) + ); + + assert_eq!( + asm("add32 r1, 2 + sub32 r1, 2 + mul32 r1, 2 + div32 r1, 2 + or32 r1, 2 + and32 r1, 2 + lsh32 r1, 2 + rsh32 r1, 2 + mod32 r1, 2 + xor32 r1, 2 + mov32 r1, 2 + arsh32 r1, 2"), + Ok(vec![ + insn(ebpf::ADD32_IMM, 1, 0, 0, 2), + insn(ebpf::SUB32_IMM, 1, 0, 0, 2), + insn(ebpf::MUL32_IMM, 1, 0, 0, 2), + insn(ebpf::DIV32_IMM, 1, 0, 0, 2), + insn(ebpf::OR32_IMM, 1, 0, 0, 2), + insn(ebpf::AND32_IMM, 1, 0, 0, 2), + insn(ebpf::LSH32_IMM, 1, 0, 0, 2), + insn(ebpf::RSH32_IMM, 1, 0, 0, 2), + insn(ebpf::MOD32_IMM, 1, 0, 0, 2), + insn(ebpf::XOR32_IMM, 1, 0, 0, 2), + insn(ebpf::MOV32_IMM, 1, 0, 0, 2), + insn(ebpf::ARSH32_IMM, 1, 0, 0, 2) + ]) + ); +} + +// Test all supported AluUnary mnemonics. +#[test] +fn test_alu_unary() { + assert_eq!( + asm("neg r1 + neg64 r1 + neg32 r1"), + Ok(vec![ + insn(ebpf::NEG64, 1, 0, 0, 0), + insn(ebpf::NEG64, 1, 0, 0, 0), + insn(ebpf::NEG32, 1, 0, 0, 0) + ]) + ); +} + +// Test all supported LoadAbs mnemonics. +#[test] +fn test_load_abs() { + assert_eq!( + asm("ldabsw 1 + ldabsh 1 + ldabsb 1 + ldabsdw 1"), + Ok(vec![ + insn(ebpf::LD_ABS_W, 0, 0, 0, 1), + insn(ebpf::LD_ABS_H, 0, 0, 0, 1), + insn(ebpf::LD_ABS_B, 0, 0, 0, 1), + insn(ebpf::LD_ABS_DW, 0, 0, 0, 1) + ]) + ); +} + +// Test all supported LoadInd mnemonics. +#[test] +fn test_load_ind() { + assert_eq!( + asm("ldindw r1, 2 + ldindh r1, 2 + ldindb r1, 2 + ldinddw r1, 2"), + Ok(vec![ + insn(ebpf::LD_IND_W, 0, 1, 0, 2), + insn(ebpf::LD_IND_H, 0, 1, 0, 2), + insn(ebpf::LD_IND_B, 0, 1, 0, 2), + insn(ebpf::LD_IND_DW, 0, 1, 0, 2) + ]) + ); +} + +// Test all supported LoadReg mnemonics. +#[test] +fn test_load_reg() { + assert_eq!( + asm("ldxw r1, [r2+3] + ldxh r1, [r2+3] + ldxb r1, [r2+3] + ldxdw r1, [r2+3]"), + Ok(vec![ + insn(ebpf::LD_W_REG, 1, 2, 3, 0), + insn(ebpf::LD_H_REG, 1, 2, 3, 0), + insn(ebpf::LD_B_REG, 1, 2, 3, 0), + insn(ebpf::LD_DW_REG, 1, 2, 3, 0) + ]) + ); +} + +// Test all supported StoreImm mnemonics. +#[test] +fn test_store_imm() { + assert_eq!( + asm("stw [r1+2], 3 + sth [r1+2], 3 + stb [r1+2], 3 + stdw [r1+2], 3"), + Ok(vec![ + insn(ebpf::ST_W_IMM, 1, 0, 2, 3), + insn(ebpf::ST_H_IMM, 1, 0, 2, 3), + insn(ebpf::ST_B_IMM, 1, 0, 2, 3), + insn(ebpf::ST_DW_IMM, 1, 0, 2, 3) + ]) + ); +} + +// Test all supported StoreReg mnemonics. +#[test] +fn test_store_reg() { + assert_eq!( + asm("stxw [r1+2], r3 + stxh [r1+2], r3 + stxb [r1+2], r3 + stxdw [r1+2], r3"), + Ok(vec![ + insn(ebpf::ST_W_REG, 1, 3, 2, 0), + insn(ebpf::ST_H_REG, 1, 3, 2, 0), + insn(ebpf::ST_B_REG, 1, 3, 2, 0), + insn(ebpf::ST_DW_REG, 1, 3, 2, 0) + ]) + ); +} + +// Test all supported JumpConditional mnemonics. +#[test] +fn test_jump_conditional() { + assert_eq!( + asm("jeq r1, r2, +3 + jgt r1, r2, +3 + jge r1, r2, +3 + jlt r1, r2, +3 + jle r1, r2, +3 + jset r1, r2, +3 + jne r1, r2, +3 + jsgt r1, r2, +3 + jsge r1, r2, +3 + jslt r1, r2, +3 + jsle r1, r2, +3"), + Ok(vec![ + insn(ebpf::JEQ_REG, 1, 2, 3, 0), + insn(ebpf::JGT_REG, 1, 2, 3, 0), + insn(ebpf::JGE_REG, 1, 2, 3, 0), + insn(ebpf::JLT_REG, 1, 2, 3, 0), + insn(ebpf::JLE_REG, 1, 2, 3, 0), + insn(ebpf::JSET_REG, 1, 2, 3, 0), + insn(ebpf::JNE_REG, 1, 2, 3, 0), + insn(ebpf::JSGT_REG, 1, 2, 3, 0), + insn(ebpf::JSGE_REG, 1, 2, 3, 0), + insn(ebpf::JSLT_REG, 1, 2, 3, 0), + insn(ebpf::JSLE_REG, 1, 2, 3, 0) + ]) + ); + + assert_eq!( + asm("jeq r1, 2, +3 + jgt r1, 2, +3 + jge r1, 2, +3 + jlt r1, 2, +3 + jle r1, 2, +3 + jset r1, 2, +3 + jne r1, 2, +3 + jsgt r1, 2, +3 + jsge r1, 2, +3 + jslt r1, 2, +3 + jsle r1, 2, +3"), + Ok(vec![ + insn(ebpf::JEQ_IMM, 1, 0, 3, 2), + insn(ebpf::JGT_IMM, 1, 0, 3, 2), + insn(ebpf::JGE_IMM, 1, 0, 3, 2), + insn(ebpf::JLT_IMM, 1, 0, 3, 2), + insn(ebpf::JLE_IMM, 1, 0, 3, 2), + insn(ebpf::JSET_IMM, 1, 0, 3, 2), + insn(ebpf::JNE_IMM, 1, 0, 3, 2), + insn(ebpf::JSGT_IMM, 1, 0, 3, 2), + insn(ebpf::JSGE_IMM, 1, 0, 3, 2), + insn(ebpf::JSLT_IMM, 1, 0, 3, 2), + insn(ebpf::JSLE_IMM, 1, 0, 3, 2) + ]) + ); + + assert_eq!( + asm("jeq32 r1, r2, +3 + jgt32 r1, r2, +3 + jge32 r1, r2, +3 + jlt32 r1, r2, +3 + jle32 r1, r2, +3 + jset32 r1, r2, +3 + jne32 r1, r2, +3 + jsgt32 r1, r2, +3 + jsge32 r1, r2, +3 + jslt32 r1, r2, +3 + jsle32 r1, r2, +3"), + Ok(vec![ + insn(ebpf::JEQ_REG32, 1, 2, 3, 0), + insn(ebpf::JGT_REG32, 1, 2, 3, 0), + insn(ebpf::JGE_REG32, 1, 2, 3, 0), + insn(ebpf::JLT_REG32, 1, 2, 3, 0), + insn(ebpf::JLE_REG32, 1, 2, 3, 0), + insn(ebpf::JSET_REG32, 1, 2, 3, 0), + insn(ebpf::JNE_REG32, 1, 2, 3, 0), + insn(ebpf::JSGT_REG32, 1, 2, 3, 0), + insn(ebpf::JSGE_REG32, 1, 2, 3, 0), + insn(ebpf::JSLT_REG32, 1, 2, 3, 0), + insn(ebpf::JSLE_REG32, 1, 2, 3, 0) + ]) + ); + + assert_eq!( + asm("jeq32 r1, 2, +3 + jgt32 r1, 2, +3 + jge32 r1, 2, +3 + jlt32 r1, 2, +3 + jle32 r1, 2, +3 + jset32 r1, 2, +3 + jne32 r1, 2, +3 + jsgt32 r1, 2, +3 + jsge32 r1, 2, +3 + jslt32 r1, 2, +3 + jsle32 r1, 2, +3"), + Ok(vec![ + insn(ebpf::JEQ_IMM32, 1, 0, 3, 2), + insn(ebpf::JGT_IMM32, 1, 0, 3, 2), + insn(ebpf::JGE_IMM32, 1, 0, 3, 2), + insn(ebpf::JLT_IMM32, 1, 0, 3, 2), + insn(ebpf::JLE_IMM32, 1, 0, 3, 2), + insn(ebpf::JSET_IMM32, 1, 0, 3, 2), + insn(ebpf::JNE_IMM32, 1, 0, 3, 2), + insn(ebpf::JSGT_IMM32, 1, 0, 3, 2), + insn(ebpf::JSGE_IMM32, 1, 0, 3, 2), + insn(ebpf::JSLT_IMM32, 1, 0, 3, 2), + insn(ebpf::JSLE_IMM32, 1, 0, 3, 2) + ]) + ); +} + +// Test all supported Endian mnemonics. +#[test] +fn test_endian() { + assert_eq!( + asm("be16 r1 + be32 r1 + be64 r1 + le16 r1 + le32 r1 + le64 r1"), + Ok(vec![ + insn(ebpf::BE, 1, 0, 0, 16), + insn(ebpf::BE, 1, 0, 0, 32), + insn(ebpf::BE, 1, 0, 0, 64), + insn(ebpf::LE, 1, 0, 0, 16), + insn(ebpf::LE, 1, 0, 0, 32), + insn(ebpf::LE, 1, 0, 0, 64) + ]) + ); +} + +#[test] +fn test_large_immediate() { + assert_eq!( + asm("add64 r1, 2147483647"), + Ok(vec![insn(ebpf::ADD64_IMM, 1, 0, 0, 2147483647)]) + ); + assert_eq!( + asm("add64 r1, -2147483648"), + Ok(vec![insn(ebpf::ADD64_IMM, 1, 0, 0, -2147483648)]) + ); +} + +#[test] +fn test_tcp_sack() { + assert_eq!(assemble(TCP_SACK_ASM), Ok(TCP_SACK_BIN.to_vec())); +} + +#[test] +fn test_error_invalid_instruction() { + assert_eq!(asm("abcd"), Err("Invalid instruction \"abcd\"".to_string())); +} + +#[test] +fn test_error_unexpected_operands() { + assert_eq!( + asm("add 1, 2"), + Err("Failed to encode add: Unexpected operands: [Integer(1), Integer(2)]".to_string()) + ); +} + +#[test] +fn test_error_too_many_operands() { + assert_eq!( + asm("add 1, 2, 3, 4"), + Err("Failed to encode add: Too many operands".to_string()) + ); +} + +#[test] +fn test_error_operands_out_of_range() { + assert_eq!( + asm("add r16, r2"), + Err("Failed to encode add: Invalid destination register 16".to_string()) + ); + assert_eq!( + asm("add r1, r16"), + Err("Failed to encode add: Invalid source register 16".to_string()) + ); + assert_eq!( + asm("ja -32769"), + Err("Failed to encode ja: Invalid offset -32769".to_string()) + ); + assert_eq!( + asm("ja 32768"), + Err("Failed to encode ja: Invalid offset 32768".to_string()) + ); + assert_eq!( + asm("add r1, 4294967296"), + Err("Failed to encode add: Invalid immediate 4294967296".to_string()) + ); + assert_eq!( + asm("add r1, 2147483648"), + Err("Failed to encode add: Invalid immediate 2147483648".to_string()) + ); + assert_eq!( + asm("add r1, -2147483649"), + Err("Failed to encode add: Invalid immediate -2147483649".to_string()) + ); +} diff --git a/kernel/crates/rbpf/tests/common.rs b/kernel/crates/rbpf/tests/common.rs new file mode 100644 index 000000000..e37839914 --- /dev/null +++ b/kernel/crates/rbpf/tests/common.rs @@ -0,0 +1,97 @@ +// SPDX-License-Identifier: (Apache-2.0 OR MIT) +// Converted from the tests for uBPF +// Copyright 2015 Big Switch Networks, Inc +// Copyright 2016 6WIND S.A. + +// Assembly code and data for tcp_sack testcases. + +#[allow(dead_code)] +pub const TCP_SACK_ASM: &str = " + ldxb r2, [r1+12] + ldxb r3, [r1+13] + lsh r3, 0x8 + or r3, r2 + mov r0, 0x0 + jne r3, 0x8, +37 + ldxb r2, [r1+23] + jne r2, 0x6, +35 + ldxb r2, [r1+14] + add r1, 0xe + and r2, 0xf + lsh r2, 0x2 + add r1, r2 + mov r0, 0x0 + ldxh r4, [r1+12] + add r1, 0x14 + rsh r4, 0x2 + and r4, 0x3c + mov r2, r4 + add r2, -20 + mov r5, 0x15 + mov r3, 0x0 + jgt r5, r4, +20 + mov r5, r3 + lsh r5, 0x20 + arsh r5, 0x20 + mov r4, r1 + add r4, r5 + ldxb r5, [r4] + jeq r5, 0x1, +4 + jeq r5, 0x0, +12 + mov r6, r3 + jeq r5, 0x5, +9 + ja +2 + add r3, 0x1 + mov r6, r3 + ldxb r3, [r4+1] + add r3, r6 + lsh r3, 0x20 + arsh r3, 0x20 + jsgt r2, r3, -18 + ja +1 + mov r0, 0x1 + exit"; + +#[allow(dead_code)] +pub const TCP_SACK_BIN: [u8; 352] = [ + 0x71, 0x12, 0x0c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x71, 0x13, 0x0d, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x67, 0x03, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x4f, 0x23, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xb7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x55, 0x03, 0x25, 0x00, 0x08, 0x00, 0x00, 0x00, + 0x71, 0x12, 0x17, 0x00, 0x00, 0x00, 0x00, 0x00, 0x55, 0x02, 0x23, 0x00, 0x06, 0x00, 0x00, 0x00, + 0x71, 0x12, 0x0e, 0x00, 0x00, 0x00, 0x00, 0x00, 0x07, 0x01, 0x00, 0x00, 0x0e, 0x00, 0x00, 0x00, + 0x57, 0x02, 0x00, 0x00, 0x0f, 0x00, 0x00, 0x00, 0x67, 0x02, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, + 0x0f, 0x21, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x69, 0x14, 0x0c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x07, 0x01, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, + 0x77, 0x04, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x57, 0x04, 0x00, 0x00, 0x3c, 0x00, 0x00, 0x00, + 0xbf, 0x42, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x07, 0x02, 0x00, 0x00, 0xec, 0xff, 0xff, 0xff, + 0xb7, 0x05, 0x00, 0x00, 0x15, 0x00, 0x00, 0x00, 0xb7, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x2d, 0x45, 0x14, 0x00, 0x00, 0x00, 0x00, 0x00, 0xbf, 0x35, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x67, 0x05, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, 0xc7, 0x05, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, + 0xbf, 0x14, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0f, 0x54, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x71, 0x45, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x15, 0x05, 0x04, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x15, 0x05, 0x0c, 0x00, 0x00, 0x00, 0x00, 0x00, 0xbf, 0x36, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x15, 0x05, 0x09, 0x00, 0x05, 0x00, 0x00, 0x00, 0x05, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x07, 0x03, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0xbf, 0x36, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x71, 0x43, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0f, 0x63, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x67, 0x03, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, 0xc7, 0x03, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, + 0x6d, 0x32, 0xee, 0xff, 0x00, 0x00, 0x00, 0x00, 0x05, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xb7, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +]; + +#[allow(dead_code)] +pub const TCP_SACK_MATCH: [u8; 78] = [ + 0x00, 0x26, 0x62, 0x2f, 0x47, 0x87, 0x00, 0x1d, 0x60, 0xb3, 0x01, 0x84, 0x08, 0x00, 0x45, 0x00, + 0x00, 0x40, 0xa8, 0xde, 0x40, 0x00, 0x40, 0x06, 0x9d, 0x58, 0xc0, 0xa8, 0x01, 0x03, 0x3f, 0x74, + 0xf3, 0x61, 0xe5, 0xc0, 0x00, 0x50, 0xe5, 0x94, 0x3f, 0x77, 0xa3, 0xc4, 0xc4, 0x80, 0xb0, 0x10, + 0x01, 0x3e, 0x34, 0xb6, 0x00, 0x00, 0x01, 0x01, 0x08, 0x0a, 0x00, 0x17, 0x95, 0x6f, 0x8d, 0x9d, + 0x9e, 0x27, 0x01, 0x01, 0x05, 0x0a, 0xa3, 0xc4, 0xca, 0x28, 0xa3, 0xc4, 0xcf, 0xd0, +]; + +#[allow(dead_code)] +pub const TCP_SACK_NOMATCH: [u8; 66] = [ + 0x00, 0x26, 0x62, 0x2f, 0x47, 0x87, 0x00, 0x1d, 0x60, 0xb3, 0x01, 0x84, 0x08, 0x00, 0x45, 0x00, + 0x00, 0x40, 0xa8, 0xde, 0x40, 0x00, 0x40, 0x06, 0x9d, 0x58, 0xc0, 0xa8, 0x01, 0x03, 0x3f, 0x74, + 0xf3, 0x61, 0xe5, 0xc0, 0x00, 0x50, 0xe5, 0x94, 0x3f, 0x77, 0xa3, 0xc4, 0xc4, 0x80, 0x80, 0x10, + 0x01, 0x3e, 0x34, 0xb6, 0x00, 0x00, 0x01, 0x01, 0x08, 0x0a, 0x00, 0x17, 0x95, 0x6f, 0x8d, 0x9d, + 0x9e, 0x27, +]; diff --git a/kernel/crates/rbpf/tests/cranelift.rs b/kernel/crates/rbpf/tests/cranelift.rs new file mode 100644 index 000000000..19aca9739 --- /dev/null +++ b/kernel/crates/rbpf/tests/cranelift.rs @@ -0,0 +1,2257 @@ +// SPDX-License-Identifier: (Apache-2.0 OR MIT) + +#![allow(clippy::unreadable_literal)] +#![cfg(feature = "cranelift")] + +extern crate rbpf; +mod common; + +use rbpf::{assembler::assemble, helpers}; + +use crate::common::{TCP_SACK_ASM, TCP_SACK_MATCH, TCP_SACK_NOMATCH}; + +macro_rules! test_cranelift { + ($name:ident, $prog:expr, $expected:expr) => { + #[test] + fn $name() { + let prog = assemble($prog).unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.cranelift_compile().unwrap(); + assert_eq!(vm.execute_program_cranelift().unwrap(), $expected); + } + }; + ($name:ident, $prog:expr, $mem:expr, $expected:expr) => { + #[test] + fn $name() { + let prog = assemble($prog).unwrap(); + let mem = &mut $mem; + let mut vm = rbpf::EbpfVmRaw::new(Some(&prog)).unwrap(); + vm.cranelift_compile().unwrap(); + assert_eq!(vm.execute_program_cranelift(mem).unwrap(), $expected); + } + }; +} + +test_cranelift!( + test_cranelift_add, + " + mov32 r0, 0 + mov32 r1, 2 + add32 r0, 1 + add32 r0, r1 + exit + ", + 0x3 +); + +test_cranelift!( + test_cranelift_alu64_arith, + " + mov r0, 0 + mov r1, 1 + mov r2, 2 + mov r3, 3 + mov r4, 4 + mov r5, 5 + mov r6, 6 + mov r7, 7 + mov r8, 8 + mov r9, 9 + add r0, 23 + add r0, r7 + sub r0, 13 + sub r0, r1 + mul r0, 7 + mul r0, r3 + div r0, 2 + div r0, r4 + exit + ", + 0x2a +); + +test_cranelift!( + test_cranelift_alu64_bit, + " + mov r0, 0 + mov r1, 1 + mov r2, 2 + mov r3, 3 + mov r4, 4 + mov r5, 5 + mov r6, 6 + mov r7, 7 + mov r8, 8 + or r0, r5 + or r0, 0xa0 + and r0, 0xa3 + mov r9, 0x91 + and r0, r9 + lsh r0, 32 + lsh r0, 22 + lsh r0, r8 + rsh r0, 32 + rsh r0, 19 + rsh r0, r7 + xor r0, 0x03 + xor r0, r2 + exit + ", + 0x11 +); + +test_cranelift!( + test_cranelift_alu_arith, + " + mov32 r0, 0 + mov32 r1, 1 + mov32 r2, 2 + mov32 r3, 3 + mov32 r4, 4 + mov32 r5, 5 + mov32 r6, 6 + mov32 r7, 7 + mov32 r8, 8 + mov32 r9, 9 + add32 r0, 23 + add32 r0, r7 + sub32 r0, 13 + sub32 r0, r1 + mul32 r0, 7 + mul32 r0, r3 + div32 r0, 2 + div32 r0, r4 + exit + ", + 0x2a +); + +test_cranelift!( + test_cranelift_alu_bit, + " + mov32 r0, 0 + mov32 r1, 1 + mov32 r2, 2 + mov32 r3, 3 + mov32 r4, 4 + mov32 r5, 5 + mov32 r6, 6 + mov32 r7, 7 + mov32 r8, 8 + or32 r0, r5 + or32 r0, 0xa0 + and32 r0, 0xa3 + mov32 r9, 0x91 + and32 r0, r9 + lsh32 r0, 22 + lsh32 r0, r8 + rsh32 r0, 19 + rsh32 r0, r7 + xor32 r0, 0x03 + xor32 r0, r2 + exit + ", + 0x11 +); + +test_cranelift!( + test_cranelift_arsh32_high_shift, + " + mov r0, 8 + lddw r1, 0x100000001 + arsh32 r0, r1 + exit + ", + 0x4 +); + +test_cranelift!( + test_cranelift_arsh, + " + mov32 r0, 0xf8 + lsh32 r0, 28 + arsh32 r0, 16 + exit + ", + 0xffff8000 +); + +test_cranelift!( + test_cranelift_arsh64, + " + mov32 r0, 1 + lsh r0, 63 + arsh r0, 55 + mov32 r1, 5 + arsh r0, r1 + exit + ", + 0xfffffffffffffff8 +); + +test_cranelift!( + test_cranelift_arsh_reg, + " + mov32 r0, 0xf8 + mov32 r1, 16 + lsh32 r0, 28 + arsh32 r0, r1 + exit + ", + 0xffff8000 +); + +test_cranelift!( + test_cranelift_be16, + " + ldxh r0, [r1] + be16 r0 + exit + ", + [0x11, 0x22], + 0x1122 +); + +test_cranelift!( + test_cranelift_be16_high, + " + ldxdw r0, [r1] + be16 r0 + exit + ", + [0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88], + 0x1122 +); + +test_cranelift!( + test_cranelift_be32, + " + ldxw r0, [r1] + be32 r0 + exit + ", + [0x11, 0x22, 0x33, 0x44], + 0x11223344 +); + +test_cranelift!( + test_cranelift_be32_high, + " + ldxdw r0, [r1] + be32 r0 + exit + ", + [0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88], + 0x11223344 +); + +test_cranelift!( + test_cranelift_be64, + " + ldxdw r0, [r1] + be64 r0 + exit + ", + [0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88], + 0x1122334455667788 +); + +#[test] +fn test_cranelift_call() { + let prog = assemble( + " + mov r1, 1 + mov r2, 2 + mov r3, 3 + mov r4, 4 + mov r5, 5 + call 0 + exit", + ) + .unwrap(); + + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.register_helper(0, helpers::gather_bytes).unwrap(); + vm.cranelift_compile().unwrap(); + assert_eq!(vm.execute_program_cranelift().unwrap(), 0x0102030405); +} + +#[test] +#[should_panic(expected = "[CRANELIFT] Error: unknown helper function (id: 0x3f)")] +fn test_cranelift_err_call_unreg() { + let prog = assemble( + " + mov r1, 1 + mov r2, 2 + mov r3, 3 + mov r4, 4 + mov r5, 5 + call 63 + exit + ", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.cranelift_compile().unwrap(); +} + +#[test] +fn test_cranelift_call_memfrob() { + let prog = assemble( + " + mov r6, r1 + add r1, 2 + mov r2, 4 + call 1 + ldxdw r0, [r6] + be64 r0 + exit", + ) + .unwrap(); + + let mut vm = rbpf::EbpfVmRaw::new(Some(&prog)).unwrap(); + vm.register_helper(1, helpers::memfrob).unwrap(); + let mem = &mut [0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08]; + vm.cranelift_compile().unwrap(); + assert_eq!( + vm.execute_program_cranelift(mem).unwrap(), + 0x102292e2f2c0708 + ); +} + +test_cranelift!( + test_cranelift_div32_high_divisor, + " + mov r0, 12 + lddw r1, 0x100000004 + div32 r0, r1 + exit + ", + 0x3 +); + +test_cranelift!( + test_cranelift_div32_imm, + " + lddw r0, 0x10000000c + div32 r0, 4 + exit + ", + 0x3 +); + +test_cranelift!( + test_cranelift_div32_reg, + " + lddw r0, 0x10000000c + mov r1, 4 + div32 r0, r1 + exit + ", + 0x3 +); + +test_cranelift!( + test_cranelift_div64_imm, + " + mov r0, 0xc + lsh r0, 32 + div r0, 4 + exit + ", + 0x300000000 +); + +test_cranelift!( + test_cranelift_div64_reg, + " + mov r0, 0xc + lsh r0, 32 + mov r1, 4 + div r0, r1 + exit + ", + 0x300000000 +); + +test_cranelift!( + test_cranelift_early_exit, + " + mov r0, 3 + exit + mov r0, 4 + exit + ", + 0x3 +); + +test_cranelift!( + test_cranelift_div64_by_zero_imm, + " + mov32 r0, 1 + div r0, 0 + exit + ", + 0x0 +); + +test_cranelift!( + test_cranelift_div_by_zero_imm, + " + mov32 r0, 1 + div32 r0, 0 + exit + ", + 0x0 +); + +test_cranelift!( + test_cranelift_mod64_by_zero_imm, + " + mov32 r0, 1 + mod r0, 0 + exit + ", + 0x1 +); + +test_cranelift!( + test_cranelift_mod_by_zero_imm, + " + mov32 r0, 1 + mod32 r0, 0 + exit + ", + 0x1 +); + +test_cranelift!( + test_cranelift_div64_by_zero_reg, + " + mov32 r0, 1 + mov32 r1, 0 + div r0, r1 + exit + ", + 0x0 +); + +test_cranelift!( + test_cranelift_div_by_zero_reg, + " + mov32 r0, 1 + mov32 r1, 0 + div32 r0, r1 + exit + ", + 0x0 +); + +test_cranelift!( + test_cranelift_mod64_by_zero_reg, + " + mov32 r0, 1 + mov32 r1, 0 + mod r0, r1 + exit + ", + 0x1 +); + +test_cranelift!( + test_cranelift_mod_by_zero_reg, + " + mov32 r0, 1 + mov32 r1, 0 + mod32 r0, r1 + exit + ", + 0x1 +); + +#[test] +// #[should_panic(expected = "Error: out of bounds memory store (insn #1)")] +#[ignore = "We have stack OOB checks, but we don't yet catch the trap code and convert it into a panic"] +fn test_cranelift_err_stack_out_of_bound() { + let prog = [ + 0x72, 0x0a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, + ]; + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.cranelift_compile().unwrap(); + vm.execute_program_cranelift().unwrap(); +} + +test_cranelift!( + test_cranelift_exit, + " + mov r0, 0 + exit + ", + 0x0 +); + +test_cranelift!( + test_cranelift_ja, + " + mov r0, 1 + ja +1 + mov r0, 2 + exit + ", + 0x1 +); + +test_cranelift!( + test_cranelift_jeq_imm, + " + mov32 r0, 0 + mov32 r1, 0xa + jeq r1, 0xb, +4 + mov32 r0, 1 + mov32 r1, 0xb + jeq r1, 0xb, +1 + mov32 r0, 2 + exit + ", + 0x1 +); + +test_cranelift!( + test_cranelift_jeq_reg, + " + mov32 r0, 0 + mov32 r1, 0xa + mov32 r2, 0xb + jeq r1, r2, +4 + mov32 r0, 1 + mov32 r1, 0xb + jeq r1, r2, +1 + mov32 r0, 2 + exit + ", + 0x1 +); + +test_cranelift!( + test_cranelift_jge_imm, + " + mov32 r0, 0 + mov32 r1, 0xa + jge r1, 0xb, +4 + mov32 r0, 1 + mov32 r1, 0xc + jge r1, 0xb, +1 + mov32 r0, 2 + exit + ", + 0x1 +); + +test_cranelift!( + test_cranelift_jle_imm, + " + mov32 r0, 0 + mov32 r1, 5 + jle r1, 4, +1 + jle r1, 6, +1 + exit + jle r1, 5, +1 + exit + mov32 r0, 1 + exit + ", + 0x1 +); + +test_cranelift!( + test_cranelift_jle_reg, + " + mov r0, 0 + mov r1, 5 + mov r2, 4 + mov r3, 6 + jle r1, r2, +2 + jle r1, r1, +1 + exit + jle r1, r3, +1 + exit + mov r0, 1 + exit + ", + 0x1 +); + +test_cranelift!( + test_cranelift_jgt_imm, + " + mov32 r0, 0 + mov32 r1, 5 + jgt r1, 6, +2 + jgt r1, 5, +1 + jgt r1, 4, +1 + exit + mov32 r0, 1 + exit + ", + 0x1 +); + +test_cranelift!( + test_cranelift_jgt_reg, + " + mov r0, 0 + mov r1, 5 + mov r2, 6 + mov r3, 4 + jgt r1, r2, +2 + jgt r1, r1, +1 + jgt r1, r3, +1 + exit + mov r0, 1 + exit + ", + 0x1 +); + +test_cranelift!( + test_cranelift_jlt_imm, + " + mov32 r0, 0 + mov32 r1, 5 + jlt r1, 4, +2 + jlt r1, 5, +1 + jlt r1, 6, +1 + exit + mov32 r0, 1 + exit + ", + 0x1 +); + +test_cranelift!( + test_cranelift_jlt_reg, + " + mov r0, 0 + mov r1, 5 + mov r2, 4 + mov r3, 6 + jlt r1, r2, +2 + jlt r1, r1, +1 + jlt r1, r3, +1 + exit + mov r0, 1 + exit + ", + 0x1 +); + +test_cranelift!( + test_cranelift_jit_bounce, + " + mov r0, 1 + mov r6, r0 + mov r7, r6 + mov r8, r7 + mov r9, r8 + mov r0, r9 + exit + ", + 0x1 +); + +test_cranelift!( + test_cranelift_jne_reg, + " + mov32 r0, 0 + mov32 r1, 0xb + mov32 r2, 0xb + jne r1, r2, +4 + mov32 r0, 1 + mov32 r1, 0xa + jne r1, r2, +1 + mov32 r0, 2 + exit + ", + 0x1 +); + +test_cranelift!( + test_cranelift_jset_imm, + " + mov32 r0, 0 + mov32 r1, 0x7 + jset r1, 0x8, +4 + mov32 r0, 1 + mov32 r1, 0x9 + jset r1, 0x8, +1 + mov32 r0, 2 + exit + ", + 0x1 +); + +test_cranelift!( + test_cranelift_jset_reg, + " + mov32 r0, 0 + mov32 r1, 0x7 + mov32 r2, 0x8 + jset r1, r2, +4 + mov32 r0, 1 + mov32 r1, 0x9 + jset r1, r2, +1 + mov32 r0, 2 + exit + ", + 0x1 +); + +test_cranelift!( + test_cranelift_jsge_imm, + " + mov32 r0, 0 + mov r1, -2 + jsge r1, -1, +5 + jsge r1, 0, +4 + mov32 r0, 1 + mov r1, -1 + jsge r1, -1, +1 + mov32 r0, 2 + exit + ", + 0x1 +); + +test_cranelift!( + test_cranelift_jsge_reg, + " + mov32 r0, 0 + mov r1, -2 + mov r2, -1 + mov32 r3, 0 + jsge r1, r2, +5 + jsge r1, r3, +4 + mov32 r0, 1 + mov r1, r2 + jsge r1, r2, +1 + mov32 r0, 2 + exit + ", + 0x1 +); + +test_cranelift!( + test_cranelift_jsle_imm, + " + mov32 r0, 0 + mov r1, -2 + jsle r1, -3, +1 + jsle r1, -1, +1 + exit + mov32 r0, 1 + jsle r1, -2, +1 + mov32 r0, 2 + exit + ", + 0x1 +); + +test_cranelift!( + test_cranelift_jsle_reg, + " + mov32 r0, 0 + mov r1, -1 + mov r2, -2 + mov32 r3, 0 + jsle r1, r2, +1 + jsle r1, r3, +1 + exit + mov32 r0, 1 + mov r1, r2 + jsle r1, r2, +1 + mov32 r0, 2 + exit + ", + 0x1 +); + +test_cranelift!( + test_cranelift_jsgt_imm, + " + mov32 r0, 0 + mov r1, -2 + jsgt r1, -1, +4 + mov32 r0, 1 + mov32 r1, 0 + jsgt r1, -1, +1 + mov32 r0, 2 + exit + ", + 0x1 +); + +test_cranelift!( + test_cranelift_jsgt_reg, + " + mov32 r0, 0 + mov r1, -2 + mov r2, -1 + jsgt r1, r2, +4 + mov32 r0, 1 + mov32 r1, 0 + jsgt r1, r2, +1 + mov32 r0, 2 + exit + ", + 0x1 +); + +test_cranelift!( + test_cranelift_jslt_imm, + " + mov32 r0, 0 + mov r1, -2 + jslt r1, -3, +2 + jslt r1, -2, +1 + jslt r1, -1, +1 + exit + mov32 r0, 1 + exit + ", + 0x1 +); + +test_cranelift!( + test_cranelift_jslt_reg, + " + mov32 r0, 0 + mov r1, -2 + mov r2, -3 + mov r3, -1 + jslt r1, r1, +2 + jslt r1, r2, +1 + jslt r1, r3, +1 + exit + mov32 r0, 1 + exit + ", + 0x1 +); + +test_cranelift!( + test_cranelift_jeq32_imm, + " + mov r9, 1 + lsh r9, 32 + mov32 r0, 0x0 + mov32 r1, 0xa + jeq32 r1, 0xb, +5 + mov32 r0, 1 + mov r1, 0xb + or r1, r9 + jeq32 r1, 0xb, +1 + mov32 r0, 2 + exit + ", + 0x1 +); + +test_cranelift!( + test_cranelift_jeq32_reg, + " + mov r9, 1 + lsh r9, 32 + mov32 r0, 0 + mov32 r1, 0xa + mov32 r2, 0xb + jeq32 r1, r2, +5 + mov32 r0, 1 + mov32 r1, 0xb + or r1, r9 + jeq32 r1, r2, +1 + mov32 r0, 2 + exit + ", + 0x1 +); + +test_cranelift!( + test_cranelift_jge32_imm, + " + mov r9, 1 + lsh r9, 32 + mov32 r0, 0 + mov32 r1, 0xa + jge32 r1, 0xb, +5 + mov32 r0, 1 + or r1, r9 + mov32 r1, 0xc + jge32 r1, 0xb, +1 + mov32 r0, 2 + exit + ", + 0x1 +); + +test_cranelift!( + test_cranelift_jge32_reg, + " + mov r9, 1 + lsh r9, 32 + mov32 r0, 0 + mov32 r1, 0xa + mov32 r2, 0xb + jge32 r1, r2, +5 + mov32 r0, 1 + or r1, r9 + mov32 r1, 0xc + jge32 r1, r2, +1 + mov32 r0, 2 + exit + ", + 0x1 +); + +test_cranelift!( + test_cranelift_jgt32_imm, + " + mov r9, 1 + lsh r9, 32 + mov32 r0, 0 + mov32 r1, 5 + or r1, r9 + jgt32 r1, 6, +4 + jgt32 r1, 5, +3 + jgt32 r1, 4, +1 + exit + mov32 r0, 1 + exit + ", + 0x1 +); + +test_cranelift!( + test_cranelift_jgt32_reg, + " + mov r9, 1 + lsh r9, 32 + mov r0, 0 + mov r1, 5 + mov32 r1, 5 + or r1, r9 + mov r2, 6 + mov r3, 4 + jgt32 r1, r2, +4 + jgt32 r1, r1, +3 + jgt32 r1, r3, +1 + exit + mov r0, 1 + exit + ", + 0x1 +); + +test_cranelift!( + test_cranelift_jle32_imm, + " + mov r9, 1 + lsh r9, 32 + mov32 r0, 0 + mov32 r1, 5 + or r1, r9 + jle32 r1, 4, +5 + jle32 r1, 6, +1 + exit + jle32 r1, 5, +1 + exit + mov32 r0, 1 + exit + ", + 0x1 +); + +test_cranelift!( + test_cranelift_jle32_reg, + " + mov r9, 1 + lsh r9, 32 + mov r0, 0 + mov r1, 5 + mov r2, 4 + mov r3, 6 + or r1, r9 + jle32 r1, r2, +5 + jle32 r1, r1, +1 + exit + jle32 r1, r3, +1 + exit + mov r0, 1 + exit + ", + 0x1 +); + +test_cranelift!( + test_cranelift_jlt32_imm, + " + mov r9, 1 + lsh r9, 32 + mov32 r0, 0 + mov32 r1, 5 + or r1, r9 + jlt32 r1, 4, +4 + jlt32 r1, 5, +3 + jlt32 r1, 6, +1 + exit + mov32 r0, 1 + exit + ", + 0x1 +); + +test_cranelift!( + test_cranelift_jlt32_reg, + " + mov r9, 1 + lsh r9, 32 + mov r0, 0 + mov r1, 5 + mov r2, 4 + mov r3, 6 + or r1, r9 + jlt32 r1, r2, +4 + jlt32 r1, r1, +3 + jlt32 r1, r3, +1 + exit + mov r0, 1 + exit + ", + 0x1 +); + +test_cranelift!( + test_cranelift_jne32_imm, + " + mov r9, 1 + lsh r9, 32 + mov32 r0, 0 + mov32 r1, 0xb + or r1, r9 + jne32 r1, 0xb, +4 + mov32 r0, 1 + mov32 r1, 0xa + or r1, r9 + jne32 r1, 0xb, +1 + mov32 r0, 2 + exit + ", + 0x1 +); + +test_cranelift!( + test_cranelift_jne32_reg, + " + mov r9, 1 + lsh r9, 32 + mov32 r0, 0 + mov32 r1, 0xb + or r1, r9 + mov32 r2, 0xb + jne32 r1, r2, +4 + mov32 r0, 1 + mov32 r1, 0xa + or r1, r9 + jne32 r1, r2, +1 + mov32 r0, 2 + exit + ", + 0x1 +); + +test_cranelift!( + test_cranelift_jset32_imm, + " + mov r9, 1 + lsh r9, 32 + mov32 r0, 0 + mov32 r1, 0x7 + or r1, r9 + jset32 r1, 0x8, +4 + mov32 r0, 1 + mov32 r1, 0x9 + jset32 r1, 0x8, +1 + mov32 r0, 2 + exit + ", + 0x1 +); + +test_cranelift!( + test_cranelift_jset32_reg, + " + mov r9, 1 + lsh r9, 32 + mov32 r0, 0 + mov32 r1, 0x7 + or r1, r9 + mov32 r2, 0x8 + jset32 r1, r2, +4 + mov32 r0, 1 + mov32 r1, 0x9 + jset32 r1, r2, +1 + mov32 r0, 2 + exit + ", + 0x1 +); + +test_cranelift!( + test_cranelift_jsge32_imm, + " + mov r9, 1 + lsh r9, 32 + mov32 r0, 0 + mov32 r1, -2 + or r1, r9 + jsge32 r1, -1, +5 + jsge32 r1, 0, +4 + mov32 r0, 1 + mov r1, -1 + jsge32 r1, -1, +1 + mov32 r0, 2 + exit + ", + 0x1 +); + +test_cranelift!( + test_cranelift_jsge32_reg, + " + mov r9, 1 + lsh r9, 32 + mov32 r0, 0 + mov32 r1, -2 + or r1, r9 + mov r2, -1 + mov32 r3, 0 + jsge32 r1, r2, +5 + jsge32 r1, r3, +4 + mov32 r0, 1 + mov r1, r2 + jsge32 r1, r2, +1 + mov32 r0, 2 + exit + ", + 0x1 +); + +test_cranelift!( + test_cranelift_jsgt32_imm, + " + mov r9, 1 + lsh r9, 32 + mov32 r0, 0 + mov32 r1, -2 + or r1, r9 + jsgt32 r1, -1, +4 + mov32 r0, 1 + mov32 r1, 0 + jsgt32 r1, -1, +1 + mov32 r0, 2 + exit + ", + 0x1 +); + +test_cranelift!( + test_cranelift_jsgt32_reg, + " + mov r9, 1 + lsh r9, 32 + mov32 r0, 0 + mov32 r1, -2 + or r1, r9 + mov r2, -1 + jsgt32 r1, r2, +4 + mov32 r0, 1 + mov32 r1, 0 + jsgt32 r1, r2, +1 + mov32 r0, 2 + exit + ", + 0x1 +); + +test_cranelift!( + test_cranelift_jsle32_imm, + " + mov r9, 1 + lsh r9, 32 + mov32 r0, 0 + mov32 r1, -2 + or r1, r9 + jsle32 r1, -3, +5 + jsle32 r1, -1, +1 + exit + mov32 r0, 1 + jsle32 r1, -2, +1 + mov32 r0, 2 + exit + ", + 0x1 +); + +test_cranelift!( + test_cranelift_jsle32_reg, + " + mov r9, 1 + lsh r9, 32 + mov32 r0, 0 + mov32 r1, -2 + or r1, r9 + mov r2, -3 + mov32 r3, 0 + jsle32 r1, r2, +6 + jsle32 r1, r3, +1 + exit + mov32 r0, 1 + mov r1, r2 + jsle32 r1, r2, +1 + mov32 r0, 2 + exit + ", + 0x1 +); + +test_cranelift!( + test_cranelift_jslt32_imm, + " + mov r9, 1 + lsh r9, 32 + mov32 r0, 0 + mov32 r1, -2 + or r1, r9 + jslt32 r1, -3, +4 + jslt32 r1, -2, +3 + jslt32 r1, -1, +1 + exit + mov32 r0, 1 + exit + ", + 0x1 +); + +test_cranelift!( + test_cranelift_jslt32_reg, + " + mov r9, 1 + lsh r9, 32 + mov32 r0, 0 + mov32 r1, -2 + or r1, r9 + mov r2, -3 + mov r3, -1 + jslt32 r1, r1, +4 + jslt32 r1, r2, +3 + jslt32 r1, r3, +1 + exit + mov32 r0, 1 + exit + ", + 0x1 +); + +test_cranelift!( + test_cranelift_lddw, + " + lddw r0, 0x1122334455667788 + exit + ", + 0x1122334455667788 +); + +test_cranelift!( + test_cranelift_lddw2, + " + lddw r0, 0x0000000080000000 + exit + ", + 0x80000000 +); + +test_cranelift!( + test_cranelift_ldxb_all, + " + mov r0, r1 + ldxb r9, [r0+0] + lsh r9, 0 + ldxb r8, [r0+1] + lsh r8, 4 + ldxb r7, [r0+2] + lsh r7, 8 + ldxb r6, [r0+3] + lsh r6, 12 + ldxb r5, [r0+4] + lsh r5, 16 + ldxb r4, [r0+5] + lsh r4, 20 + ldxb r3, [r0+6] + lsh r3, 24 + ldxb r2, [r0+7] + lsh r2, 28 + ldxb r1, [r0+8] + lsh r1, 32 + ldxb r0, [r0+9] + lsh r0, 36 + or r0, r1 + or r0, r2 + or r0, r3 + or r0, r4 + or r0, r5 + or r0, r6 + or r0, r7 + or r0, r8 + or r0, r9 + exit + ", + [0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09], + 0x9876543210 +); + +test_cranelift!( + test_cranelift_ldxb, + " + ldxb r0, [r1+2] + exit + ", + [0xaa, 0xbb, 0x11, 0xcc, 0xdd], + 0x11 +); + +test_cranelift!( + test_cranelift_ldxdw, + " + ldxdw r0, [r1+2] + exit + ", + [0xaa, 0xbb, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88, 0xcc, 0xdd], + 0x8877665544332211 +); + +test_cranelift!( + test_cranelift_ldxh_all, + " + mov r0, r1 + ldxh r9, [r0+0] + be16 r9 + lsh r9, 0 + ldxh r8, [r0+2] + be16 r8 + lsh r8, 4 + ldxh r7, [r0+4] + be16 r7 + lsh r7, 8 + ldxh r6, [r0+6] + be16 r6 + lsh r6, 12 + ldxh r5, [r0+8] + be16 r5 + lsh r5, 16 + ldxh r4, [r0+10] + be16 r4 + lsh r4, 20 + ldxh r3, [r0+12] + be16 r3 + lsh r3, 24 + ldxh r2, [r0+14] + be16 r2 + lsh r2, 28 + ldxh r1, [r0+16] + be16 r1 + lsh r1, 32 + ldxh r0, [r0+18] + be16 r0 + lsh r0, 36 + or r0, r1 + or r0, r2 + or r0, r3 + or r0, r4 + or r0, r5 + or r0, r6 + or r0, r7 + or r0, r8 + or r0, r9 + exit + ", + [ + 0x00, 0x00, 0x00, 0x01, 0x00, 0x02, 0x00, 0x03, 0x00, 0x04, 0x00, 0x05, 0x00, 0x06, 0x00, + 0x07, 0x00, 0x08, 0x00, 0x09 + ], + 0x9876543210 +); + +test_cranelift!( + test_cranelift_ldxh_all2, + " + mov r0, r1 + ldxh r9, [r0+0] + be16 r9 + ldxh r8, [r0+2] + be16 r8 + ldxh r7, [r0+4] + be16 r7 + ldxh r6, [r0+6] + be16 r6 + ldxh r5, [r0+8] + be16 r5 + ldxh r4, [r0+10] + be16 r4 + ldxh r3, [r0+12] + be16 r3 + ldxh r2, [r0+14] + be16 r2 + ldxh r1, [r0+16] + be16 r1 + ldxh r0, [r0+18] + be16 r0 + or r0, r1 + or r0, r2 + or r0, r3 + or r0, r4 + or r0, r5 + or r0, r6 + or r0, r7 + or r0, r8 + or r0, r9 + exit + ", + [ + 0x00, 0x01, 0x00, 0x02, 0x00, 0x04, 0x00, 0x08, 0x00, 0x10, 0x00, 0x20, 0x00, 0x40, 0x00, + 0x80, 0x01, 0x00, 0x02, 0x00 + ], + 0x3ff +); + +test_cranelift!( + test_cranelift_ldxh, + " + ldxh r0, [r1+2] + exit + ", + [0xaa, 0xbb, 0x11, 0x22, 0xcc, 0xdd], + 0x2211 +); + +test_cranelift!( + test_cranelift_ldxh_same_reg, + " + mov r0, r1 + sth [r0], 0x1234 + ldxh r0, [r0] + exit + ", + [0xff, 0xff], + 0x1234 +); + +test_cranelift!( + test_cranelift_ldxw_all, + " + mov r0, r1 + ldxw r9, [r0+0] + be32 r9 + ldxw r8, [r0+4] + be32 r8 + ldxw r7, [r0+8] + be32 r7 + ldxw r6, [r0+12] + be32 r6 + ldxw r5, [r0+16] + be32 r5 + ldxw r4, [r0+20] + be32 r4 + ldxw r3, [r0+24] + be32 r3 + ldxw r2, [r0+28] + be32 r2 + ldxw r1, [r0+32] + be32 r1 + ldxw r0, [r0+36] + be32 r0 + or r0, r1 + or r0, r2 + or r0, r3 + or r0, r4 + or r0, r5 + or r0, r6 + or r0, r7 + or r0, r8 + or r0, r9 + exit + ", + [ + 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x08, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x08, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00 + ], + 0x030f0f +); + +test_cranelift!( + test_cranelift_ldxw, + " + ldxw r0, [r1+2] + exit + ", + [0xaa, 0xbb, 0x11, 0x22, 0x33, 0x44, 0xcc, 0xdd], + 0x44332211 +); + +test_cranelift!( + test_cranelift_le16, + " + ldxh r0, [r1] + le16 r0 + exit + ", + [0x22, 0x11], + 0x1122 +); + +test_cranelift!( + test_cranelift_le32, + " + ldxw r0, [r1] + le32 r0 + exit + ", + [0x44, 0x33, 0x22, 0x11], + 0x11223344 +); + +test_cranelift!( + test_cranelift_le64, + " + ldxdw r0, [r1] + le64 r0 + exit + ", + [0x88, 0x77, 0x66, 0x55, 0x44, 0x33, 0x22, 0x11], + 0x1122334455667788 +); + +test_cranelift!( + test_cranelift_lsh_reg, + " + mov r0, 0x1 + mov r7, 4 + lsh r0, r7 + exit + ", + 0x10 +); + +test_cranelift!( + test_cranelift_mod, + " + mov32 r0, 5748 + mod32 r0, 92 + mov32 r1, 13 + mod32 r0, r1 + exit + ", + 0x5 +); + +test_cranelift!( + test_cranelift_mod32, + " + lddw r0, 0x100000003 + mod32 r0, 3 + exit + ", + 0x0 +); + +test_cranelift!( + test_cranelift_mod64, + " + mov32 r0, -1316649930 + lsh r0, 32 + or r0, 0x100dc5c8 + mov32 r1, 0xdde263e + lsh r1, 32 + or r1, 0x3cbef7f3 + mod r0, r1 + mod r0, 0x658f1778 + exit + ", + 0x30ba5a04 +); + +test_cranelift!( + test_cranelift_mov, + " + mov32 r1, 1 + mov32 r0, r1 + exit + ", + 0x1 +); + +test_cranelift!( + test_cranelift_mul32_imm, + " + mov r0, 3 + mul32 r0, 4 + exit + ", + 0xc +); + +test_cranelift!( + test_cranelift_mul32_reg, + " + mov r0, 3 + mov r1, 4 + mul32 r0, r1 + exit + ", + 0xc +); + +test_cranelift!( + test_cranelift_mul32_reg_overflow, + " + mov r0, 0x40000001 + mov r1, 4 + mul32 r0, r1 + exit + ", + 0x4 +); + +test_cranelift!( + test_cranelift_mul64_imm, + " + mov r0, 0x40000001 + mul r0, 4 + exit + ", + 0x100000004 +); + +test_cranelift!( + test_cranelift_mul64_reg, + " + mov r0, 0x40000001 + mov r1, 4 + mul r0, r1 + exit + ", + 0x100000004 +); + +test_cranelift!( + test_cranelift_mul_loop, + " + mov r0, 0x7 + add r1, 0xa + lsh r1, 0x20 + rsh r1, 0x20 + jeq r1, 0x0, +4 + mov r0, 0x7 + mul r0, 0x7 + add r1, -1 + jne r1, 0x0, -3 + exit + ", + 0x75db9c97 +); + +test_cranelift!( + test_cranelift_neg64, + " + mov32 r0, 2 + neg r0 + exit + ", + 0xfffffffffffffffe +); + +test_cranelift!( + test_cranelift_neg, + " + mov32 r0, 2 + neg32 r0 + exit + ", + 0xfffffffe +); + +test_cranelift!( + test_cranelift_prime, + " + mov r1, 67 + mov r0, 0x1 + mov r2, 0x2 + jgt r1, 0x2, +4 + ja +10 + add r2, 0x1 + mov r0, 0x1 + jge r2, r1, +7 + mov r3, r1 + div r3, r2 + mul r3, r2 + mov r4, r1 + sub r4, r3 + mov r0, 0x0 + jne r4, 0x0, -10 + exit + ", + 1 +); + +test_cranelift!( + test_cranelift_rhs32, + " + xor r0, r0 + sub r0, 1 + rsh32 r0, 8 + exit + ", + 0x00ffffff +); + +test_cranelift!( + test_cranelift_rsh_reg, + " + mov r0, 0x10 + mov r7, 4 + rsh r0, r7 + exit + ", + 0x1 +); + +test_cranelift!( + test_cranelift_stack, + " + mov r1, 51 + stdw [r10-16], 0xab + stdw [r10-8], 0xcd + and r1, 1 + lsh r1, 3 + mov r2, r10 + add r2, r1 + ldxdw r0, [r2-16] + exit + ", + 0xcd +); + +#[test] +fn test_cranelift_stack2() { + let prog = assemble( + " + stb [r10-4], 0x01 + stb [r10-3], 0x02 + stb [r10-2], 0x03 + stb [r10-1], 0x04 + mov r1, r10 + mov r2, 0x4 + sub r1, r2 + call 1 + mov r1, 0 + ldxb r2, [r10-4] + ldxb r3, [r10-3] + ldxb r4, [r10-2] + ldxb r5, [r10-1] + call 0 + xor r0, 0x2a2a2a2a + exit", + ) + .unwrap(); + + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.register_helper(0, helpers::gather_bytes).unwrap(); + vm.register_helper(1, helpers::memfrob).unwrap(); + vm.cranelift_compile().unwrap(); + assert_eq!(vm.execute_program_cranelift().unwrap(), 0x01020304); +} + +test_cranelift!( + test_cranelift_stb, + " + stb [r1+2], 0x11 + ldxb r0, [r1+2] + exit + ", + [0xaa, 0xbb, 0xff, 0xcc, 0xdd], + 0x11 +); + +test_cranelift!( + test_cranelift_stdw, + " + stdw [r1+2], 0x44332211 + ldxdw r0, [r1+2] + exit + ", + [0xaa, 0xbb, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xcc, 0xdd], + 0x44332211 +); + +test_cranelift!( + test_cranelift_sth, + " + sth [r1+2], 0x2211 + ldxh r0, [r1+2] + exit + ", + [0xaa, 0xbb, 0xff, 0xff, 0xcc, 0xdd], + 0x2211 +); + +#[test] +#[ignore] +fn test_cranelift_string_stack() { + let prog = assemble( + " + mov r1, 0x78636261 + stxw [r10-8], r1 + mov r6, 0x0 + stxb [r10-4], r6 + stxb [r10-12], r6 + mov r1, 0x79636261 + stxw [r10-16], r1 + mov r1, r10 + add r1, -8 + mov r2, r1 + call 0x4 + mov r1, r0 + mov r0, 0x1 + lsh r1, 0x20 + rsh r1, 0x20 + jne r1, 0x0, +11 + mov r1, r10 + add r1, -8 + mov r2, r10 + add r2, -16 + call 0x4 + mov r1, r0 + lsh r1, 0x20 + rsh r1, 0x20 + mov r0, 0x1 + jeq r1, r6, +1 + mov r0, 0x0 + exit", + ) + .unwrap(); + + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.register_helper(4, helpers::strcmp).unwrap(); + vm.cranelift_compile().unwrap(); + assert_eq!(vm.execute_program_cranelift().unwrap(), 0x0); +} + +test_cranelift!( + test_cranelift_stw, + " + stw [r1+2], 0x44332211 + ldxw r0, [r1+2] + exit + ", + [0xaa, 0xbb, 0xff, 0xff, 0xff, 0xff, 0xcc, 0xdd], + 0x44332211 +); + +test_cranelift!( + test_cranelift_stxb, + " + mov32 r2, 0x11 + stxb [r1+2], r2 + ldxb r0, [r1+2] + exit + ", + [0xaa, 0xbb, 0xff, 0xcc, 0xdd], + 0x11 +); + +test_cranelift!( + test_cranelift_stxb_all, + " + mov r0, 0xf0 + mov r2, 0xf2 + mov r3, 0xf3 + mov r4, 0xf4 + mov r5, 0xf5 + mov r6, 0xf6 + mov r7, 0xf7 + mov r8, 0xf8 + stxb [r1], r0 + stxb [r1+1], r2 + stxb [r1+2], r3 + stxb [r1+3], r4 + stxb [r1+4], r5 + stxb [r1+5], r6 + stxb [r1+6], r7 + stxb [r1+7], r8 + ldxdw r0, [r1] + be64 r0 + exit + ", + [0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff], + 0xf0f2f3f4f5f6f7f8 +); + +test_cranelift!( + test_cranelift_stxb_all2, + " + mov r0, r1 + mov r1, 0xf1 + mov r9, 0xf9 + stxb [r0], r1 + stxb [r0+1], r9 + ldxh r0, [r0] + be16 r0 + exit + ", + [0xff, 0xff], + 0xf1f9 +); + +test_cranelift!( + test_cranelift_stxb_chain, + " + mov r0, r1 + ldxb r9, [r0+0] + stxb [r0+1], r9 + ldxb r8, [r0+1] + stxb [r0+2], r8 + ldxb r7, [r0+2] + stxb [r0+3], r7 + ldxb r6, [r0+3] + stxb [r0+4], r6 + ldxb r5, [r0+4] + stxb [r0+5], r5 + ldxb r4, [r0+5] + stxb [r0+6], r4 + ldxb r3, [r0+6] + stxb [r0+7], r3 + ldxb r2, [r0+7] + stxb [r0+8], r2 + ldxb r1, [r0+8] + stxb [r0+9], r1 + ldxb r0, [r0+9] + exit + ", + [0x2a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00], + 0x2a +); + +test_cranelift!( + test_cranelift_stxdw, + " + mov r2, -2005440939 + lsh r2, 32 + or r2, 0x44332211 + stxdw [r1+2], r2 + ldxdw r0, [r1+2] + exit + ", + [0xaa, 0xbb, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xcc, 0xdd], + 0x8877665544332211 +); + +test_cranelift!( + test_cranelift_stxh, + " + mov32 r2, 0x2211 + stxh [r1+2], r2 + ldxh r0, [r1+2] + exit + ", + [0xaa, 0xbb, 0xff, 0xff, 0xcc, 0xdd], + 0x2211 +); + +test_cranelift!( + test_cranelift_stxw, + " + mov32 r2, 0x44332211 + stxw [r1+2], r2 + ldxw r0, [r1+2] + exit + ", + [0xaa, 0xbb, 0xff, 0xff, 0xff, 0xff, 0xcc, 0xdd], + 0x44332211 +); + +test_cranelift!( + test_cranelift_subnet, + " + mov r2, 0xe + ldxh r3, [r1+12] + jne r3, 0x81, +2 + mov r2, 0x12 + ldxh r3, [r1+16] + and r3, 0xffff + jne r3, 0x8, +5 + add r1, r2 + mov r0, 0x1 + ldxw r1, [r1+16] + and r1, 0xffffff + jeq r1, 0x1a8c0, +1 + mov r0, 0x0 + exit + ", + [ + 0x00, 0x00, 0xc0, 0x9f, 0xa0, 0x97, 0x00, 0xa0, 0xcc, 0x3b, 0xbf, 0xfa, 0x08, 0x00, 0x45, + 0x10, 0x00, 0x3c, 0x46, 0x3c, 0x40, 0x00, 0x40, 0x06, 0x73, 0x1c, 0xc0, 0xa8, 0x01, 0x02, + 0xc0, 0xa8, 0x01, 0x01, 0x06, 0x0e, 0x00, 0x17, 0x99, 0xc5, 0xa0, 0xec, 0x00, 0x00, 0x00, + 0x00, 0xa0, 0x02, 0x7d, 0x78, 0xe0, 0xa3, 0x00, 0x00, 0x02, 0x04, 0x05, 0xb4, 0x04, 0x02, + 0x08, 0x0a, 0x00, 0x9c, 0x27, 0x24, 0x00, 0x00, 0x00, 0x00, 0x01, 0x03, 0x03, 0x00, + ], + 0x1 +); + +const PROG_TCP_PORT_80: [u8; 152] = [ + 0x71, 0x12, 0x0c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x71, 0x13, 0x0d, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x67, 0x03, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x4f, 0x23, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xb7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x55, 0x03, 0x0c, 0x00, 0x08, 0x00, 0x00, 0x00, + 0x71, 0x12, 0x17, 0x00, 0x00, 0x00, 0x00, 0x00, 0x55, 0x02, 0x0a, 0x00, 0x06, 0x00, 0x00, 0x00, + 0x71, 0x12, 0x0e, 0x00, 0x00, 0x00, 0x00, 0x00, 0x07, 0x01, 0x00, 0x00, 0x0e, 0x00, 0x00, 0x00, + 0x57, 0x02, 0x00, 0x00, 0x0f, 0x00, 0x00, 0x00, 0x67, 0x02, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, + 0x0f, 0x21, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x69, 0x12, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x15, 0x02, 0x02, 0x00, 0x00, 0x50, 0x00, 0x00, 0x69, 0x11, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x55, 0x01, 0x01, 0x00, 0x00, 0x50, 0x00, 0x00, 0xb7, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +]; + +#[test] +fn test_cranelift_tcp_port80_match() { + let mem = &mut [ + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x00, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x08, 0x00, 0x45, + 0x00, 0x00, 0x56, 0x00, 0x01, 0x00, 0x00, 0x40, 0x06, 0xf9, 0x4d, 0xc0, 0xa8, 0x00, 0x01, + 0xc0, 0xa8, 0x00, 0x02, 0x27, 0x10, 0x00, 0x50, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x50, 0x02, 0x20, 0x00, 0xc5, 0x18, 0x00, 0x00, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, + 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, + 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, + 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, + ]; + let prog = &PROG_TCP_PORT_80; + let mut vm = rbpf::EbpfVmRaw::new(Some(prog)).unwrap(); + vm.cranelift_compile().unwrap(); + assert_eq!(vm.execute_program_cranelift(mem).unwrap(), 0x1); +} + +#[test] +fn test_cranelift_tcp_port80_nomatch() { + let mem = &mut [ + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x00, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x08, 0x00, 0x45, + 0x00, 0x00, 0x56, 0x00, 0x01, 0x00, 0x00, 0x40, 0x06, 0xf9, 0x4d, 0xc0, 0xa8, 0x00, 0x01, + 0xc0, 0xa8, 0x00, 0x02, 0x00, 0x16, 0x27, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x51, 0x02, 0x20, 0x00, 0xc5, 0x18, 0x00, 0x00, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, + 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, + 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, + 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, + ]; + let prog = &PROG_TCP_PORT_80; + let mut vm = rbpf::EbpfVmRaw::new(Some(prog)).unwrap(); + vm.cranelift_compile().unwrap(); + assert_eq!(vm.execute_program_cranelift(mem).unwrap(), 0x0); +} + +#[test] +fn test_cranelift_tcp_port80_nomatch_ethertype() { + let mem = &mut [ + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x00, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x08, 0x01, 0x45, + 0x00, 0x00, 0x56, 0x00, 0x01, 0x00, 0x00, 0x40, 0x06, 0xf9, 0x4d, 0xc0, 0xa8, 0x00, 0x01, + 0xc0, 0xa8, 0x00, 0x02, 0x27, 0x10, 0x00, 0x50, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x50, 0x02, 0x20, 0x00, 0xc5, 0x18, 0x00, 0x00, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, + 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, + 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, + 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, + ]; + let prog = &PROG_TCP_PORT_80; + let mut vm = rbpf::EbpfVmRaw::new(Some(prog)).unwrap(); + vm.cranelift_compile().unwrap(); + assert_eq!(vm.execute_program_cranelift(mem).unwrap(), 0x0); +} + +#[test] +fn test_cranelift_tcp_port80_nomatch_proto() { + let mem = &mut [ + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x00, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x08, 0x00, 0x45, + 0x00, 0x00, 0x56, 0x00, 0x01, 0x00, 0x00, 0x40, 0x11, 0xf9, 0x4d, 0xc0, 0xa8, 0x00, 0x01, + 0xc0, 0xa8, 0x00, 0x02, 0x27, 0x10, 0x00, 0x50, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x50, 0x02, 0x20, 0x00, 0xc5, 0x18, 0x00, 0x00, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, + 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, + 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, + 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, + ]; + let prog = &PROG_TCP_PORT_80; + let mut vm = rbpf::EbpfVmRaw::new(Some(prog)).unwrap(); + vm.cranelift_compile().unwrap(); + assert_eq!(vm.execute_program_cranelift(mem).unwrap(), 0x0); +} + +#[test] +fn test_cranelift_tcp_sack_match() { + let mut mem = TCP_SACK_MATCH.to_vec(); + let prog = assemble(TCP_SACK_ASM).unwrap(); + let mut vm = rbpf::EbpfVmRaw::new(Some(&prog)).unwrap(); + vm.cranelift_compile().unwrap(); + assert_eq!( + vm.execute_program_cranelift(mem.as_mut_slice()).unwrap(), + 0x1 + ); +} + +#[test] +fn test_cranelift_tcp_sack_nomatch() { + let mut mem = TCP_SACK_NOMATCH.to_vec(); + let prog = assemble(TCP_SACK_ASM).unwrap(); + let mut vm = rbpf::EbpfVmRaw::new(Some(&prog)).unwrap(); + vm.cranelift_compile().unwrap(); + assert_eq!( + vm.execute_program_cranelift(mem.as_mut_slice()).unwrap(), + 0x0 + ); +} + +#[test] +fn test_cranelift_ldabsb() { + let prog = &[ + 0x30, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, + ]; + let mem = &mut [ + 0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88, 0x99, 0xaa, 0xbb, 0xcc, 0xdd, 0xee, + 0xff, + ]; + let mut vm = rbpf::EbpfVmFixedMbuff::new(Some(prog), 0x00, 0x08).unwrap(); + + vm.cranelift_compile().unwrap(); + assert_eq!(vm.execute_program_cranelift(mem).unwrap(), 0x33); +} + +#[test] +fn test_cranelift_ldabsh() { + let prog = &[ + 0x28, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, + ]; + let mem = &mut [ + 0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88, 0x99, 0xaa, 0xbb, 0xcc, 0xdd, 0xee, + 0xff, + ]; + let mut vm = rbpf::EbpfVmFixedMbuff::new(Some(prog), 0x00, 0x08).unwrap(); + + vm.cranelift_compile().unwrap(); + assert_eq!(vm.execute_program_cranelift(mem).unwrap(), 0x4433); +} + +#[test] +fn test_cranelift_ldabsw() { + let prog = &[ + 0x20, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, + ]; + let mem = &mut [ + 0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88, 0x99, 0xaa, 0xbb, 0xcc, 0xdd, 0xee, + 0xff, + ]; + let mut vm = rbpf::EbpfVmFixedMbuff::new(Some(prog), 0x00, 0x08).unwrap(); + + vm.cranelift_compile().unwrap(); + assert_eq!(vm.execute_program_cranelift(mem).unwrap(), 0x66554433); +} + +#[test] +fn test_cranelift_ldabsdw() { + let prog = &[ + 0x38, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, + ]; + let mem = &mut [ + 0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88, 0x99, 0xaa, 0xbb, 0xcc, 0xdd, 0xee, + 0xff, + ]; + let mut vm = rbpf::EbpfVmFixedMbuff::new(Some(prog), 0x00, 0x08).unwrap(); + + vm.cranelift_compile().unwrap(); + assert_eq!( + vm.execute_program_cranelift(mem).unwrap(), + 0xaa99887766554433 + ); +} + +#[test] +fn test_cranelift_ldindb() { + let prog = &[ + 0xb7, 0x01, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, 0x50, 0x10, 0x00, 0x00, 0x03, 0x00, 0x00, + 0x00, 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + ]; + let mem = &mut [ + 0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88, 0x99, 0xaa, 0xbb, 0xcc, 0xdd, 0xee, + 0xff, + ]; + let mut vm = rbpf::EbpfVmFixedMbuff::new(Some(prog), 0x00, 0x08).unwrap(); + + vm.cranelift_compile().unwrap(); + assert_eq!(vm.execute_program_cranelift(mem).unwrap(), 0x88); +} + +#[test] +fn test_cranelift_ldindh() { + let prog = &[ + 0xb7, 0x01, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, 0x48, 0x10, 0x00, 0x00, 0x03, 0x00, 0x00, + 0x00, 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + ]; + let mem = &mut [ + 0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88, 0x99, 0xaa, 0xbb, 0xcc, 0xdd, 0xee, + 0xff, + ]; + let mut vm = rbpf::EbpfVmFixedMbuff::new(Some(prog), 0x00, 0x08).unwrap(); + + vm.cranelift_compile().unwrap(); + assert_eq!(vm.execute_program_cranelift(mem).unwrap(), 0x9988); +} + +#[test] +fn test_cranelift_ldindw() { + let prog = &[ + 0xb7, 0x01, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x40, 0x10, 0x00, 0x00, 0x01, 0x00, 0x00, + 0x00, 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + ]; + let mem = &mut [ + 0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88, 0x99, 0xaa, 0xbb, 0xcc, 0xdd, 0xee, + 0xff, + ]; + let mut vm = rbpf::EbpfVmFixedMbuff::new(Some(prog), 0x00, 0x08).unwrap(); + + vm.cranelift_compile().unwrap(); + assert_eq!(vm.execute_program_cranelift(mem).unwrap(), 0x88776655); +} + +#[test] +fn test_cranelift_ldinddw() { + let prog = &[ + 0xb7, 0x01, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x58, 0x10, 0x00, 0x00, 0x03, 0x00, 0x00, + 0x00, 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + ]; + let mem = &mut [ + 0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88, 0x99, 0xaa, 0xbb, 0xcc, 0xdd, 0xee, + 0xff, + ]; + let mut vm = rbpf::EbpfVmFixedMbuff::new(Some(prog), 0x00, 0x08).unwrap(); + + vm.cranelift_compile().unwrap(); + assert_eq!( + vm.execute_program_cranelift(mem).unwrap(), + 0xccbbaa9988776655 + ); +} diff --git a/kernel/crates/rbpf/tests/disassembler.rs b/kernel/crates/rbpf/tests/disassembler.rs new file mode 100644 index 000000000..90f7ebe42 --- /dev/null +++ b/kernel/crates/rbpf/tests/disassembler.rs @@ -0,0 +1,377 @@ +// SPDX-License-Identifier: (Apache-2.0 OR MIT) +// Copyright 2017 Jan-Erik Rediger +// +// Adopted from tests in `tests/assembler.rs` + +extern crate rbpf; +mod common; + +use rbpf::{assembler::assemble, disassembler::to_insn_vec}; + +// Using a macro to keep actual line numbers in failure output +macro_rules! disasm { + ($src:expr) => {{ + let src = $src; + let asm = assemble(src).expect("Can't assemble from string"); + let insn = to_insn_vec(&asm); + let reasm = insn + .into_iter() + .map(|ins| ins.desc) + .collect::>() + .join("\n"); + + assert_eq!(src, reasm); + }}; +} + +#[test] +fn test_empty() { + disasm!(""); +} + +// Example for InstructionType::NoOperand. +#[test] +fn test_exit() { + disasm!("exit"); +} + +// Example for InstructionType::AluBinary. +#[test] +fn test_add64() { + disasm!("add64 r1, r3"); + disasm!("add64 r1, 0x5"); +} + +// Example for InstructionType::AluUnary. +#[test] +fn test_neg64() { + disasm!("neg64 r1"); +} + +// Example for InstructionType::LoadReg. +#[test] +fn test_ldxw() { + disasm!("ldxw r1, [r2+0x5]"); +} + +// Example for InstructionType::StoreImm. +#[test] +fn test_stw() { + disasm!("stw [r2+0x5], 0x7"); +} + +// Example for InstructionType::StoreReg. +#[test] +fn test_stxw() { + disasm!("stxw [r2+0x5], r8"); +} + +// Example for InstructionType::JumpUnconditional. +#[test] +fn test_ja() { + disasm!("ja +0x8"); +} + +// Example for InstructionType::JumpConditional. +#[test] +fn test_jeq() { + disasm!("jeq r1, 0x4, +0x8"); + disasm!("jeq r1, r3, +0x8"); +} + +// Example for InstructionType::Call. +#[test] +fn test_call() { + disasm!("call 0x3"); +} + +// Example for InstructionType::Endian. +#[test] +fn test_be32() { + disasm!("be32 r1"); +} + +// Example for InstructionType::LoadImm. +#[test] +fn test_lddw() { + disasm!("lddw r1, 0x1234abcd5678eeff"); + disasm!("lddw r1, 0xff11ee22dd33cc44"); +} + +// Example for InstructionType::LoadAbs. +#[test] +fn test_ldabsw() { + disasm!("ldabsw 0x1"); +} + +// Example for InstructionType::LoadInd. +#[test] +fn test_ldindw() { + disasm!("ldindw r1, 0x2"); +} + +// Example for InstructionType::LoadReg. +#[test] +fn test_ldxdw() { + disasm!("ldxdw r1, [r2+0x3]"); +} + +// Example for InstructionType::StoreImm. +#[test] +fn test_sth() { + disasm!("sth [r1+0x2], 0x3"); +} + +// Example for InstructionType::StoreReg. +#[test] +fn test_stxh() { + disasm!("stxh [r1+0x2], r3"); +} + +// Test all supported AluBinary mnemonics. +#[test] +fn test_alu_binary() { + disasm!( + "add64 r1, r2 +sub64 r1, r2 +mul64 r1, r2 +div64 r1, r2 +or64 r1, r2 +and64 r1, r2 +lsh64 r1, r2 +rsh64 r1, r2 +mod64 r1, r2 +xor64 r1, r2 +mov64 r1, r2 +arsh64 r1, r2" + ); + + disasm!( + "add64 r1, 0x2 +sub64 r1, 0x2 +mul64 r1, 0x2 +div64 r1, 0x2 +or64 r1, 0x2 +and64 r1, 0x2 +lsh64 r1, 0x2 +rsh64 r1, 0x2 +mod64 r1, 0x2 +xor64 r1, 0x2 +mov64 r1, 0x2 +arsh64 r1, 0x2" + ); + + disasm!( + "add32 r1, r2 +sub32 r1, r2 +mul32 r1, r2 +div32 r1, r2 +or32 r1, r2 +and32 r1, r2 +lsh32 r1, r2 +rsh32 r1, r2 +mod32 r1, r2 +xor32 r1, r2 +mov32 r1, r2 +arsh32 r1, r2" + ); + + disasm!( + "add32 r1, 0x2 +sub32 r1, 0x2 +mul32 r1, 0x2 +div32 r1, 0x2 +or32 r1, 0x2 +and32 r1, 0x2 +lsh32 r1, 0x2 +rsh32 r1, 0x2 +mod32 r1, 0x2 +xor32 r1, 0x2 +mov32 r1, 0x2 +arsh32 r1, 0x2" + ); +} + +// Test all supported AluUnary mnemonics. +#[test] +fn test_alu_unary() { + disasm!( + "neg64 r1 +neg32 r1" + ); +} + +// Test all supported LoadAbs mnemonics. +#[test] +fn test_load_abs() { + disasm!( + "ldabsw 0x1 +ldabsh 0x1 +ldabsb 0x1 +ldabsdw 0x1" + ); +} + +// Test all supported LoadInd mnemonics. +#[test] +fn test_load_ind() { + disasm!( + "ldindw r1, 0x2 +ldindh r1, 0x2 +ldindb r1, 0x2 +ldinddw r1, 0x2" + ); +} + +// Test all supported LoadReg mnemonics. +#[test] +fn test_load_reg() { + disasm!( + r"ldxw r1, [r2+0x3] +ldxh r1, [r2+0x3] +ldxb r1, [r2+0x3] +ldxdw r1, [r2+0x3]" + ); +} + +// Test all supported StoreImm mnemonics. +#[test] +fn test_store_imm() { + disasm!( + "stw [r1+0x2], 0x3 +sth [r1+0x2], 0x3 +stb [r1+0x2], 0x3 +stdw [r1+0x2], 0x3" + ); +} + +// Test all supported StoreReg mnemonics. +#[test] +fn test_store_reg() { + disasm!( + "stxw [r1+0x2], r3 +stxh [r1+0x2], r3 +stxb [r1+0x2], r3 +stxdw [r1+0x2], r3" + ); +} + +// Test all supported JumpConditional mnemonics. +#[test] +fn test_jump_conditional() { + disasm!( + "jeq r1, r2, +0x3 +jgt r1, r2, +0x3 +jge r1, r2, +0x3 +jlt r1, r2, +0x3 +jle r1, r2, +0x3 +jset r1, r2, +0x3 +jne r1, r2, +0x3 +jsgt r1, r2, +0x3 +jsge r1, r2, -0x3 +jslt r1, r2, +0x3 +jsle r1, r2, -0x3" + ); + + disasm!( + "jeq r1, 0x2, +0x3 +jgt r1, 0x2, +0x3 +jge r1, 0x2, +0x3 +jlt r1, 0x2, +0x3 +jle r1, 0x2, +0x3 +jset r1, 0x2, +0x3 +jne r1, 0x2, +0x3 +jsgt r1, 0x2, +0x3 +jsge r1, 0x2, -0x3 +jslt r1, 0x2, +0x3 +jsle r1, 0x2, -0x3" + ); + + disasm!( + "jeq32 r1, r2, +0x3 +jgt32 r1, r2, +0x3 +jge32 r1, r2, +0x3 +jlt32 r1, r2, +0x3 +jle32 r1, r2, +0x3 +jset32 r1, r2, +0x3 +jne32 r1, r2, +0x3 +jsgt32 r1, r2, +0x3 +jsge32 r1, r2, -0x3 +jslt32 r1, r2, +0x3 +jsle32 r1, r2, -0x3" + ); + + disasm!( + "jeq32 r1, 0x2, +0x3 +jgt32 r1, 0x2, +0x3 +jge32 r1, 0x2, +0x3 +jlt32 r1, 0x2, +0x3 +jle32 r1, 0x2, +0x3 +jset32 r1, 0x2, +0x3 +jne32 r1, 0x2, +0x3 +jsgt32 r1, 0x2, +0x3 +jsge32 r1, 0x2, -0x3 +jslt32 r1, 0x2, +0x3 +jsle32 r1, 0x2, -0x3" + ); +} + +// Test all supported Endian mnemonics. +#[test] +fn test_endian() { + disasm!( + "be16 r1 +be32 r1 +be64 r1 +le16 r1 +le32 r1 +le64 r1" + ); +} + +#[test] +fn test_large_immediate() { + disasm!("add64 r1, 0x7fffffff"); + disasm!("add64 r1, 0x7fffffff"); +} + +// Non-regression tests for overflow when trying to negate offset 0x8000i16. +#[test] +fn test_offset_overflow() { + let insns = [ + 0x62, 0x01, 0x00, 0x80, 0x01, 0x00, 0x00, 0x00, // stw + 0x6a, 0x01, 0x00, 0x80, 0x01, 0x00, 0x00, 0x00, // sth + 0x72, 0x01, 0x00, 0x80, 0x01, 0x00, 0x00, 0x00, // stb + 0x7a, 0x01, 0x00, 0x80, 0x01, 0x00, 0x00, 0x00, // stdw + 0x61, 0x01, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, // ldxw + 0x69, 0x01, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, // ldxh + 0x71, 0x01, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, // ldxb + 0x79, 0x01, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, // ldxdw + 0x15, 0x01, 0x00, 0x80, 0x02, 0x00, 0x00, 0x00, // jeq (imm) + 0x1d, 0x21, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, // jeq (reg) + 0x16, 0x01, 0x00, 0x80, 0x02, 0x00, 0x00, 0x00, // jeq32 (imm) + 0x1e, 0x21, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, // jeq32 (reg) + ]; + + let expected_output = "stw [r1-0x8000], 0x1 +sth [r1-0x8000], 0x1 +stb [r1-0x8000], 0x1 +stdw [r1-0x8000], 0x1 +ldxw r1, [r0-0x8000] +ldxh r1, [r0-0x8000] +ldxb r1, [r0-0x8000] +ldxdw r1, [r0-0x8000] +jeq r1, 0x2, -0x8000 +jeq r1, r2, -0x8000 +jeq32 r1, 0x2, -0x8000 +jeq32 r1, r2, -0x8000"; + + let prog = to_insn_vec(&insns); + let asm = prog + .into_iter() + .map(|ins| ins.desc) + .collect::>() + .join("\n"); + + assert_eq!(asm, expected_output); +} diff --git a/kernel/crates/rbpf/tests/misc.rs b/kernel/crates/rbpf/tests/misc.rs new file mode 100644 index 000000000..10bb053d0 --- /dev/null +++ b/kernel/crates/rbpf/tests/misc.rs @@ -0,0 +1,571 @@ +// SPDX-License-Identifier: (Apache-2.0 OR MIT) +// Copyright 2016 6WIND S.A. + +// There are unused mut warnings due to unsafe code. +#![allow(unused_mut)] +#![allow(clippy::unreadable_literal)] + +// This crate would be needed to load bytecode from a BPF-compiled object file. Since the crate +// is not used anywhere else in the library, it is deactivated: we do not want to load and compile +// it just for the tests. If you want to use it, do not forget to add the following +// dependency to your Cargo.toml file: +// +// --- +// elf = "0.0.10" +// --- +// +// extern crate elf; +// use std::path::PathBuf; + +extern crate rbpf; + +#[cfg(feature = "std")] +use rbpf::helpers; +use rbpf::{assembler::assemble, Error, ErrorKind}; + +// The following two examples have been compiled from C with the following command: +// +// ```bash +// clang -O2 -emit-llvm -c -o - | llc -march=bpf -filetype=obj -o +// ``` +// +// The C source code was the following: +// +// ```c +// #include +// #include +// #include +// #include +// +// #define ETH_ALEN 6 +// #define ETH_P_IP 0x0008 /* htons(0x0800) */ +// #define TCP_HDR_LEN 20 +// +// #define BLOCKED_TCP_PORT 0x9999 +// +// struct eth_hdr { +// unsigned char h_dest[ETH_ALEN]; +// unsigned char h_source[ETH_ALEN]; +// unsigned short h_proto; +// }; +// +// #define SEC(NAME) __attribute__((section(NAME), used)) +// SEC(".classifier") +// int handle_ingress(struct __sk_buff *skb) +// { +// void *data = (void *)(long)skb->data; +// void *data_end = (void *)(long)skb->data_end; +// struct eth_hdr *eth = data; +// struct iphdr *iph = data + sizeof(*eth); +// struct tcphdr *tcp = data + sizeof(*eth) + sizeof(*iph); +// +// /* single length check */ +// if (data + sizeof(*eth) + sizeof(*iph) + sizeof(*tcp) > data_end) +// return 0; +// if (eth->h_proto != ETH_P_IP) +// return 0; +// if (iph->protocol != IPPROTO_TCP) +// return 0; +// if (tcp->source == BLOCKED_TCP_PORT || tcp->dest == BLOCKED_TCP_PORT) +// return -1; +// return 0; +// } +// char _license[] SEC(".license") = "GPL"; +// ``` +// +// This program, once compiled, can be injected into Linux kernel, with tc for instance. Sadly, we +// need to bring some modifications to the generated bytecode in order to run it: the three +// instructions with opcode 0x61 load data from a packet area as 4-byte words, where we need to +// load it as 8-bytes double words (0x79). The kernel does the same kind of translation before +// running the program, but rbpf does not implement this. +// +// In addition, the offset at which the pointer to the packet data is stored must be changed: since +// we use 8 bytes instead of 4 for the start and end addresses of the data packet, we cannot use +// the offsets produced by clang (0x4c and 0x50), the addresses would overlap. Instead we can use, +// for example, 0x40 and 0x50. See comments on the bytecode below to see the modifications. +// +// Once the bytecode has been (manually, in our case) edited, we can load the bytecode directly +// from the ELF object file. This is easy to do, but requires the addition of two crates in the +// Cargo.toml file (see comments above), so here we use just the hardcoded bytecode instructions +// instead. + +#[test] +#[cfg(feature = "std")] +fn test_vm_block_port() { + // To load the bytecode from an object file instead of using the hardcoded instructions, + // use the additional crates commented at the beginning of this file (and also add them to your + // Cargo.toml). See comments above. + // + // --- + // let filename = "my_ebpf_object_file.o"; + // + // let path = PathBuf::from(filename); + // let file = match elf::File::open_path(&path) { + // Ok(f) => f, + // Err(e) => panic!("Error: {:?}", e), + // }; + // + // let text_scn = match file.get_section(".classifier") { + // Some(s) => s, + // None => panic!("Failed to look up .classifier section"), + // }; + // + // let prog = &text_scn.data; + // --- + + let prog = &[ + 0xb7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x79, 0x12, 0x50, 0x00, 0x00, 0x00, 0x00, + 0x00, // 0x79 instead of 0x61 + 0x79, 0x11, 0x40, 0x00, 0x00, 0x00, 0x00, + 0x00, // 0x79 instead of 0x61, 0x40 i.o. 0x4c + 0xbf, 0x13, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x07, 0x03, 0x00, 0x00, 0x36, 0x00, 0x00, + 0x00, 0x2d, 0x23, 0x12, 0x00, 0x00, 0x00, 0x00, 0x00, 0x69, 0x12, 0x0c, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x55, 0x02, 0x10, 0x00, 0x08, 0x00, 0x00, 0x00, 0x71, 0x12, 0x17, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x55, 0x02, 0x0e, 0x00, 0x06, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x79, 0x11, 0x22, + 0x00, 0x00, 0x00, 0x00, 0x00, // 0x79 instead of 0x61 + 0xbf, 0x12, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x57, 0x02, 0x00, 0x00, 0xff, 0xff, 0x00, + 0x00, 0x15, 0x02, 0x08, 0x00, 0x99, 0x99, 0x00, 0x00, 0x18, 0x02, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x5f, 0x21, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0xb7, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x18, 0x02, 0x00, 0x00, + 0x00, 0x00, 0x99, 0x99, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1d, 0x21, 0x01, + 0x00, 0x00, 0x00, 0x00, 0x00, 0xb7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x95, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + ]; + + let packet = &mut [ + 0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xfe, 0xdc, 0xba, 0x98, 0x76, 0x54, 0x08, + 0x00, // ethertype + 0x45, 0x00, 0x00, 0x3b, // start ip_hdr + 0xa6, 0xab, 0x40, 0x00, 0x40, 0x06, 0x96, 0x0f, 0x7f, 0x00, 0x00, 0x01, 0x7f, 0x00, 0x00, + 0x01, + // Program matches the next two bytes: 0x9999 returns 0xffffffff, else return 0. + 0x99, 0x99, 0xc6, 0xcc, // start tcp_hdr + 0xd1, 0xe5, 0xc4, 0x9d, 0xd4, 0x30, 0xb5, 0xd2, 0x80, 0x18, 0x01, 0x56, 0xfe, 0x2f, 0x00, + 0x00, 0x01, 0x01, 0x08, 0x0a, // start data + 0x00, 0x23, 0x75, 0x89, 0x00, 0x23, 0x63, 0x2d, 0x71, 0x64, 0x66, 0x73, 0x64, 0x66, 0x0au8, + ]; + + let mut vm = rbpf::EbpfVmFixedMbuff::new(Some(prog), 0x40, 0x50).unwrap(); + vm.register_helper(helpers::BPF_TRACE_PRINTK_IDX, helpers::bpf_trace_printf) + .unwrap(); + + let res = vm.execute_program(packet).unwrap(); + println!("Program returned: {res:?} ({res:#x})"); + assert_eq!(res, 0xffffffff); +} + +#[test] +#[cfg(all(not(windows), feature = "std"))] +fn test_jit_block_port() { + // To load the bytecode from an object file instead of using the hardcoded instructions, + // use the additional crates commented at the beginning of this file (and also add them to your + // Cargo.toml). See comments above. + // + // --- + // let filename = "my_ebpf_object_file.o"; + // + // let path = PathBuf::from(filename); + // let file = match elf::File::open_path(&path) { + // Ok(f) => f, + // Err(e) => panic!("Error: {:?}", e), + // }; + // + // let text_scn = match file.get_section(".classifier") { + // Some(s) => s, + // None => panic!("Failed to look up .classifier section"), + // }; + // + // let prog = &text_scn.data; + // --- + + let prog = &[ + 0xb7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x79, 0x12, 0x50, 0x00, 0x00, 0x00, 0x00, + 0x00, // 0x79 instead of 0x61 + 0x79, 0x11, 0x40, 0x00, 0x00, 0x00, 0x00, + 0x00, // 0x79 instead of 0x61, 0x40 i.o. 0x4c + 0xbf, 0x13, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x07, 0x03, 0x00, 0x00, 0x36, 0x00, 0x00, + 0x00, 0x2d, 0x23, 0x12, 0x00, 0x00, 0x00, 0x00, 0x00, 0x69, 0x12, 0x0c, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x55, 0x02, 0x10, 0x00, 0x08, 0x00, 0x00, 0x00, 0x71, 0x12, 0x17, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x55, 0x02, 0x0e, 0x00, 0x06, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x79, 0x11, 0x22, + 0x00, 0x00, 0x00, 0x00, 0x00, // 0x79 instead of 0x61 + 0xbf, 0x12, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x57, 0x02, 0x00, 0x00, 0xff, 0xff, 0x00, + 0x00, 0x15, 0x02, 0x08, 0x00, 0x99, 0x99, 0x00, 0x00, 0x18, 0x02, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x5f, 0x21, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0xb7, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x18, 0x02, 0x00, 0x00, + 0x00, 0x00, 0x99, 0x99, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1d, 0x21, 0x01, + 0x00, 0x00, 0x00, 0x00, 0x00, 0xb7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x95, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + ]; + + let packet = &mut [ + 0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xfe, 0xdc, 0xba, 0x98, 0x76, 0x54, 0x08, + 0x00, // ethertype + 0x45, 0x00, 0x00, 0x3b, // start ip_hdr + 0xa6, 0xab, 0x40, 0x00, 0x40, 0x06, 0x96, 0x0f, 0x7f, 0x00, 0x00, 0x01, 0x7f, 0x00, 0x00, + 0x01, + // Program matches the next two bytes: 0x9999 returns 0xffffffff, else return 0. + 0x99, 0x99, 0xc6, 0xcc, // start tcp_hdr + 0xd1, 0xe5, 0xc4, 0x9d, 0xd4, 0x30, 0xb5, 0xd2, 0x80, 0x18, 0x01, 0x56, 0xfe, 0x2f, 0x00, + 0x00, 0x01, 0x01, 0x08, 0x0a, // start data + 0x00, 0x23, 0x75, 0x89, 0x00, 0x23, 0x63, 0x2d, 0x71, 0x64, 0x66, 0x73, 0x64, 0x66, 0x0au8, + ]; + + let mut vm = rbpf::EbpfVmFixedMbuff::new(Some(prog), 0x40, 0x50).unwrap(); + vm.register_helper(helpers::BPF_TRACE_PRINTK_IDX, helpers::bpf_trace_printf) + .unwrap(); + vm.jit_compile().unwrap(); + + unsafe { + let res = vm.execute_program_jit(packet).unwrap(); + println!("Program returned: {res:?} ({res:#x})"); + assert_eq!(res, 0xffffffff); + } +} + +// Program and memory come from uBPF test ldxh. +#[test] +fn test_vm_mbuff() { + let prog = &[ + // Load mem from mbuff into R1 + 0x79, 0x11, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, // ldhx r1[2], r0 + 0x69, 0x10, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, + ]; + let mem = &[0xaa, 0xbb, 0x11, 0x22, 0xcc, 0xdd]; + + let mbuff = [0u8; 32]; + unsafe { + let mut data = mbuff.as_ptr().offset(8) as *mut u64; + let mut data_end = mbuff.as_ptr().offset(24) as *mut u64; + data.write_unaligned(mem.as_ptr() as u64); + data_end.write_unaligned(mem.as_ptr() as u64 + mem.len() as u64); + } + + let vm = rbpf::EbpfVmMbuff::new(Some(prog)).unwrap(); + assert_eq!(vm.execute_program(mem, &mbuff).unwrap(), 0x2211); +} + +// Program and memory come from uBPF test ldxh. +#[test] +fn test_vm_mbuff_with_rust_api() { + use rbpf::insn_builder::*; + + let mut program = BpfCode::new(); + program + .load_x(MemSize::DoubleWord) + .set_dst(0x01) + .set_src(0x01) + .set_off(0x00_08) + .push() + .load_x(MemSize::HalfWord) + .set_dst(0x00) + .set_src(0x01) + .set_off(0x00_02) + .push() + .exit() + .push(); + + let mem = &[0xaa, 0xbb, 0x11, 0x22, 0xcc, 0xdd]; + + let mbuff = [0u8; 32]; + unsafe { + let mut data = mbuff.as_ptr().offset(8) as *mut u64; + let mut data_end = mbuff.as_ptr().offset(24) as *mut u64; + *data = mem.as_ptr() as u64; + *data_end = mem.as_ptr() as u64 + mem.len() as u64; + } + + let vm = rbpf::EbpfVmMbuff::new(Some(program.into_bytes())).unwrap(); + assert_eq!(vm.execute_program(mem, &mbuff).unwrap(), 0x2211); +} + +// Program and memory come from uBPF test ldxh. +#[test] +#[cfg(all(not(windows), feature = "std"))] +fn test_jit_mbuff() { + let prog = &[ + // Load mem from mbuff into R1 + 0x79, 0x11, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, // ldhx r1[2], r0 + 0x69, 0x10, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, + ]; + let mem = &mut [0xaa, 0xbb, 0x11, 0x22, 0xcc, 0xdd]; + + let mut mbuff = [0u8; 32]; + unsafe { + let mut data = mbuff.as_ptr().offset(8) as *mut u64; + let mut data_end = mbuff.as_ptr().offset(24) as *mut u64; + *data = mem.as_ptr() as u64; + *data_end = mem.as_ptr() as u64 + mem.len() as u64; + } + + unsafe { + let mut vm = rbpf::EbpfVmMbuff::new(Some(prog)).unwrap(); + vm.jit_compile().unwrap(); + assert_eq!(vm.execute_program_jit(mem, &mut mbuff).unwrap(), 0x2211); + } +} + +#[cfg(all(not(windows), feature = "std"))] +#[test] +fn test_vm_jit_ldabsb() { + let prog = &[ + 0x30, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, + ]; + let mem = &mut [ + 0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88, 0x99, 0xaa, 0xbb, 0xcc, 0xdd, 0xee, + 0xff, + ]; + let mut vm = rbpf::EbpfVmRaw::new(Some(prog)).unwrap(); + assert_eq!(vm.execute_program(mem).unwrap(), 0x33); + + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit(mem).unwrap(), 0x33); + }; +} + +#[cfg(all(not(windows), feature = "std"))] +#[test] +fn test_vm_jit_ldabsh() { + let prog = &[ + 0x28, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, + ]; + let mem = &mut [ + 0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88, 0x99, 0xaa, 0xbb, 0xcc, 0xdd, 0xee, + 0xff, + ]; + let mut vm = rbpf::EbpfVmRaw::new(Some(prog)).unwrap(); + assert_eq!(vm.execute_program(mem).unwrap(), 0x4433); + + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit(mem).unwrap(), 0x4433); + }; +} + +#[cfg(all(not(windows), feature = "std"))] +#[test] +fn test_vm_jit_ldabsw() { + let prog = &[ + 0x20, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, + ]; + let mem = &mut [ + 0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88, 0x99, 0xaa, 0xbb, 0xcc, 0xdd, 0xee, + 0xff, + ]; + let mut vm = rbpf::EbpfVmRaw::new(Some(prog)).unwrap(); + assert_eq!(vm.execute_program(mem).unwrap(), 0x66554433); + vm.jit_compile().unwrap(); + + unsafe { + assert_eq!(vm.execute_program_jit(mem).unwrap(), 0x66554433); + }; +} + +#[cfg(all(not(windows), feature = "std"))] +#[test] +fn test_vm_jit_ldabsdw() { + let prog = &[ + 0x38, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, + ]; + let mem = &mut [ + 0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88, 0x99, 0xaa, 0xbb, 0xcc, 0xdd, 0xee, + 0xff, + ]; + let mut vm = rbpf::EbpfVmRaw::new(Some(prog)).unwrap(); + assert_eq!(vm.execute_program(mem).unwrap(), 0xaa99887766554433); + vm.jit_compile().unwrap(); + + unsafe { + assert_eq!(vm.execute_program_jit(mem).unwrap(), 0xaa99887766554433); + }; +} + +#[test] +#[should_panic(expected = "Error: out of bounds memory load (insn #1),")] +fn test_vm_err_ldabsb_oob() { + let prog = &[ + 0x38, 0x00, 0x00, 0x00, 0x33, 0x00, 0x00, 0x00, 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, + ]; + let mem = &mut [ + 0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88, 0x99, 0xaa, 0xbb, 0xcc, 0xdd, 0xee, + 0xff, + ]; + let vm = rbpf::EbpfVmRaw::new(Some(prog)).unwrap(); + vm.execute_program(mem).unwrap(); + + // Memory check not implemented for JIT yet. +} + +#[test] +#[should_panic(expected = "Error: out of bounds memory load (insn #1),")] +fn test_vm_err_ldabsb_nomem() { + let prog = &[ + 0x38, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, + ]; + let vm = rbpf::EbpfVmNoData::new(Some(prog)).unwrap(); + vm.execute_program().unwrap(); + + // Memory check not implemented for JIT yet. +} + +#[cfg(all(not(windows), feature = "std"))] +#[test] +fn test_vm_jit_ldindb() { + let prog = &[ + 0xb7, 0x01, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, 0x50, 0x10, 0x00, 0x00, 0x03, 0x00, 0x00, + 0x00, 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + ]; + let mem = &mut [ + 0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88, 0x99, 0xaa, 0xbb, 0xcc, 0xdd, 0xee, + 0xff, + ]; + let mut vm = rbpf::EbpfVmRaw::new(Some(prog)).unwrap(); + assert_eq!(vm.execute_program(mem).unwrap(), 0x88); + + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit(mem).unwrap(), 0x88); + }; +} + +#[cfg(all(not(windows), feature = "std"))] +#[test] +fn test_vm_jit_ldindh() { + let prog = &[ + 0xb7, 0x01, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, 0x48, 0x10, 0x00, 0x00, 0x03, 0x00, 0x00, + 0x00, 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + ]; + let mem = &mut [ + 0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88, 0x99, 0xaa, 0xbb, 0xcc, 0xdd, 0xee, + 0xff, + ]; + let mut vm = rbpf::EbpfVmRaw::new(Some(prog)).unwrap(); + assert_eq!(vm.execute_program(mem).unwrap(), 0x9988); + + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit(mem).unwrap(), 0x9988); + }; +} + +#[cfg(all(not(windows), feature = "std"))] +#[test] +fn test_vm_jit_ldindw() { + let prog = &[ + 0xb7, 0x01, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x40, 0x10, 0x00, 0x00, 0x01, 0x00, 0x00, + 0x00, 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + ]; + let mem = &mut [ + 0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88, 0x99, 0xaa, 0xbb, 0xcc, 0xdd, 0xee, + 0xff, + ]; + let mut vm = rbpf::EbpfVmRaw::new(Some(prog)).unwrap(); + assert_eq!(vm.execute_program(mem).unwrap(), 0x88776655); + vm.jit_compile().unwrap(); + + unsafe { + assert_eq!(vm.execute_program_jit(mem).unwrap(), 0x88776655); + }; +} + +#[cfg(all(not(windows), feature = "std"))] +#[test] +fn test_vm_jit_ldinddw() { + let prog = &[ + 0xb7, 0x01, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x58, 0x10, 0x00, 0x00, 0x03, 0x00, 0x00, + 0x00, 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + ]; + let mem = &mut [ + 0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88, 0x99, 0xaa, 0xbb, 0xcc, 0xdd, 0xee, + 0xff, + ]; + let mut vm = rbpf::EbpfVmRaw::new(Some(prog)).unwrap(); + assert_eq!(vm.execute_program(mem).unwrap(), 0xccbbaa9988776655); + vm.jit_compile().unwrap(); + + unsafe { + assert_eq!(vm.execute_program_jit(mem).unwrap(), 0xccbbaa9988776655); + }; +} + +#[test] +#[should_panic(expected = "Error: out of bounds memory load (insn #2),")] +fn test_vm_err_ldindb_oob() { + let prog = &[ + 0xb7, 0x01, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, 0x38, 0x10, 0x00, 0x00, 0x33, 0x00, 0x00, + 0x00, 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + ]; + let mem = &mut [ + 0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88, 0x99, 0xaa, 0xbb, 0xcc, 0xdd, 0xee, + 0xff, + ]; + let vm = rbpf::EbpfVmRaw::new(Some(prog)).unwrap(); + vm.execute_program(mem).unwrap(); + + // Memory check not implemented for JIT yet. +} + +#[test] +#[should_panic(expected = "Error: out of bounds memory load (insn #2),")] +fn test_vm_err_ldindb_nomem() { + let prog = &[ + 0xb7, 0x01, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x38, 0x10, 0x00, 0x00, 0x03, 0x00, 0x00, + 0x00, 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + ]; + let vm = rbpf::EbpfVmNoData::new(Some(prog)).unwrap(); + vm.execute_program().unwrap(); + + // Memory check not implemented for JIT yet. +} + +#[test] +#[should_panic(expected = "Error: No program set, call prog_set() to load one")] +fn test_vm_exec_no_program() { + let vm = rbpf::EbpfVmNoData::new(None).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0xBEE); +} + +fn verifier_success(_prog: &[u8]) -> Result<(), Error> { + Ok(()) +} + +fn verifier_fail(_prog: &[u8]) -> Result<(), Error> { + Err(Error::new(ErrorKind::Other, "Gaggablaghblagh!")) +} + +#[test] +fn test_verifier_success() { + let prog = assemble( + "mov32 r0, 0xBEE + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(None).unwrap(); + vm.set_verifier(verifier_success).unwrap(); + vm.set_program(&prog).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0xBEE); +} + +#[test] +#[should_panic(expected = "Gaggablaghblagh!")] +fn test_verifier_fail() { + let prog = assemble( + "mov32 r0, 0xBEE + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(None).unwrap(); + vm.set_verifier(verifier_fail).unwrap(); + vm.set_program(&prog).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0xBEE); +} diff --git a/kernel/crates/rbpf/tests/ubpf_jit_x86_64.rs b/kernel/crates/rbpf/tests/ubpf_jit_x86_64.rs new file mode 100644 index 000000000..2740a0ae6 --- /dev/null +++ b/kernel/crates/rbpf/tests/ubpf_jit_x86_64.rs @@ -0,0 +1,2891 @@ +// SPDX-License-Identifier: (Apache-2.0 OR MIT) +// Converted from the tests for uBPF +// Copyright 2015 Big Switch Networks, Inc +// Copyright 2016 6WIND S.A. + +// The tests contained in this file are extracted from the unit tests of uBPF software. Each test +// in this file has a name in the form `test_jit_`, and corresponds to the (human-readable) +// code in `ubpf/tree/master/tests/`, available at +// (hyphen had to be replaced with underscores +// as Rust will not accept them in function names). It is strongly advised to refer to the uBPF +// version to understand what these program do. +// +// Each program was assembled from the uBPF version with the assembler provided by uBPF itself, and +// available at . +// The very few modifications that have been realized should be indicated. + +// These are unit tests for the eBPF JIT compiler. + +#![allow(clippy::unreadable_literal)] +#![cfg(all(not(windows), feature = "std"))] + +extern crate rbpf; +mod common; + +use common::{TCP_SACK_ASM, TCP_SACK_MATCH, TCP_SACK_NOMATCH}; +use rbpf::{assembler::assemble, helpers}; + +#[test] +fn test_jit_add() { + let prog = assemble( + " + mov32 r0, 0 + mov32 r1, 2 + add32 r0, 1 + add32 r0, r1 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x3); + } +} + +#[test] +fn test_jit_alu64_arith() { + let prog = assemble( + " + mov r0, 0 + mov r1, 1 + mov r2, 2 + mov r3, 3 + mov r4, 4 + mov r5, 5 + mov r6, 6 + mov r7, 7 + mov r8, 8 + mov r9, 9 + add r0, 23 + add r0, r7 + sub r0, 13 + sub r0, r1 + mul r0, 7 + mul r0, r3 + div r0, 2 + div r0, r4 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x2a); + } +} + +#[test] +fn test_jit_alu64_bit() { + let prog = assemble( + " + mov r0, 0 + mov r1, 1 + mov r2, 2 + mov r3, 3 + mov r4, 4 + mov r5, 5 + mov r6, 6 + mov r7, 7 + mov r8, 8 + or r0, r5 + or r0, 0xa0 + and r0, 0xa3 + mov r9, 0x91 + and r0, r9 + lsh r0, 32 + lsh r0, 22 + lsh r0, r8 + rsh r0, 32 + rsh r0, 19 + rsh r0, r7 + xor r0, 0x03 + xor r0, r2 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x11); + } +} + +#[test] +fn test_jit_alu_arith() { + let prog = assemble( + " + mov32 r0, 0 + mov32 r1, 1 + mov32 r2, 2 + mov32 r3, 3 + mov32 r4, 4 + mov32 r5, 5 + mov32 r6, 6 + mov32 r7, 7 + mov32 r8, 8 + mov32 r9, 9 + add32 r0, 23 + add32 r0, r7 + sub32 r0, 13 + sub32 r0, r1 + mul32 r0, 7 + mul32 r0, r3 + div32 r0, 2 + div32 r0, r4 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x2a); + } +} + +#[test] +fn test_jit_alu_bit() { + let prog = assemble( + " + mov32 r0, 0 + mov32 r1, 1 + mov32 r2, 2 + mov32 r3, 3 + mov32 r4, 4 + mov32 r5, 5 + mov32 r6, 6 + mov32 r7, 7 + mov32 r8, 8 + or32 r0, r5 + or32 r0, 0xa0 + and32 r0, 0xa3 + mov32 r9, 0x91 + and32 r0, r9 + lsh32 r0, 22 + lsh32 r0, r8 + rsh32 r0, 19 + rsh32 r0, r7 + xor32 r0, 0x03 + xor32 r0, r2 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x11); + } +} + +#[test] +fn test_jit_arsh32_high_shift() { + let prog = assemble( + " + mov r0, 8 + lddw r1, 0x100000001 + arsh32 r0, r1 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x4); + } +} + +#[test] +fn test_jit_arsh() { + let prog = assemble( + " + mov32 r0, 0xf8 + lsh32 r0, 28 + arsh32 r0, 16 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0xffff8000); + } +} + +#[test] +fn test_jit_arsh64() { + let prog = assemble( + " + mov32 r0, 1 + lsh r0, 63 + arsh r0, 55 + mov32 r1, 5 + arsh r0, r1 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0xfffffffffffffff8); + } +} + +#[test] +fn test_jit_arsh_reg() { + let prog = assemble( + " + mov32 r0, 0xf8 + mov32 r1, 16 + lsh32 r0, 28 + arsh32 r0, r1 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0xffff8000); + } +} + +#[test] +fn test_jit_be16() { + let prog = assemble( + " + ldxh r0, [r1] + be16 r0 + exit", + ) + .unwrap(); + let mem = &mut [0x11, 0x22]; + let mut vm = rbpf::EbpfVmRaw::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit(mem).unwrap(), 0x1122); + } +} + +#[test] +fn test_jit_be16_high() { + let prog = assemble( + " + ldxdw r0, [r1] + be16 r0 + exit", + ) + .unwrap(); + let mem = &mut [0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88]; + let mut vm = rbpf::EbpfVmRaw::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit(mem).unwrap(), 0x1122); + } +} + +#[test] +fn test_jit_be32() { + let prog = assemble( + " + ldxw r0, [r1] + be32 r0 + exit", + ) + .unwrap(); + let mem = &mut [0x11, 0x22, 0x33, 0x44]; + let mut vm = rbpf::EbpfVmRaw::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit(mem).unwrap(), 0x11223344); + } +} + +#[test] +fn test_jit_be32_high() { + let prog = assemble( + " + ldxdw r0, [r1] + be32 r0 + exit", + ) + .unwrap(); + let mem = &mut [0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88]; + let mut vm = rbpf::EbpfVmRaw::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit(mem).unwrap(), 0x11223344); + } +} + +#[test] +fn test_jit_be64() { + let prog = assemble( + " + ldxdw r0, [r1] + be64 r0 + exit", + ) + .unwrap(); + let mem = &mut [0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88]; + let mut vm = rbpf::EbpfVmRaw::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit(mem).unwrap(), 0x1122334455667788); + } +} + +#[test] +fn test_jit_call() { + let prog = assemble( + " + mov r1, 1 + mov r2, 2 + mov r3, 3 + mov r4, 4 + mov r5, 5 + call 0 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.register_helper(0, helpers::gather_bytes).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x0102030405); + } +} + +#[test] +fn test_jit_call_memfrob() { + let prog = assemble( + " + mov r6, r1 + add r1, 2 + mov r2, 4 + call 1 + ldxdw r0, [r6] + be64 r0 + exit", + ) + .unwrap(); + let mem = &mut [0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08]; + let mut vm = rbpf::EbpfVmRaw::new(Some(&prog)).unwrap(); + vm.register_helper(1, helpers::memfrob).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit(mem).unwrap(), 0x102292e2f2c0708); + } +} + +// TODO: helpers::trash_registers needs asm!(). +// Try this again once asm!() is available in stable. +//#[test] +//fn test_jit_call_save() { +//let prog = &[ +//0xb7, 0x06, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, +//0xb7, 0x07, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, +//0xb7, 0x08, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, +//0xb7, 0x09, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, +//0x85, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, +//0xb7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +//0x4f, 0x60, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +//0x4f, 0x70, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +//0x4f, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +//0x4f, 0x90, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +//0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 +//]; +//let mut vm = rbpf::EbpfVmNoData::new(Some(prog)).unwrap(); +//vm.register_helper(2, helpers::trash_registers); +//vm.jit_compile().unwrap(); +//unsafe { assert_eq!(vm.execute_program_jit().unwrap(), 0x4321); } +//} + +#[test] +fn test_jit_div32_high_divisor() { + let prog = assemble( + " + mov r0, 12 + lddw r1, 0x100000004 + div32 r0, r1 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x3); + } +} + +#[test] +fn test_jit_div32_imm() { + let prog = assemble( + " + lddw r0, 0x10000000c + div32 r0, 4 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x3); + } +} + +#[test] +fn test_jit_div32_reg() { + let prog = assemble( + " + lddw r0, 0x10000000c + mov r1, 4 + div32 r0, r1 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x3); + } +} + +#[test] +fn test_jit_div64_imm() { + let prog = assemble( + " + mov r0, 0xc + lsh r0, 32 + div r0, 4 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x300000000); + } +} + +#[test] +fn test_jit_div64_reg() { + let prog = assemble( + " + mov r0, 0xc + lsh r0, 32 + mov r1, 4 + div r0, r1 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x300000000); + } +} + +// For some register numbers, we don't emit the same instructions for handling divisions by zero, +// which means we don't use the same offset to skip these instructions when the divisor is not +// zero. We've had a regression because of this before, make sure we test it. +#[test] +fn test_jit_div32_highreg() { + let prog = assemble( + " + mov r0, 2 + mov r7, 4 + div32 r7, r0 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x2); + } +} + +#[test] +fn test_jit_div64_highreg() { + let prog = assemble( + " + mov r0, 2 + mov r7, 4 + div r7, r0 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x2); + } +} + +#[test] +fn test_jit_early_exit() { + let prog = assemble( + " + mov r0, 3 + exit + mov r0, 4 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x3); + } +} + +// uBPF limits the number of user functions at 64. We don't. +//#[test] +//fn test_jit_err_call_bad_imm() { +//} + +#[test] +#[should_panic(expected = "[JIT] Error: unknown helper function (id: 0x3f)")] +fn test_jit_err_call_unreg() { + let prog = assemble( + " + mov r1, 1 + mov r2, 2 + mov r3, 3 + mov r4, 4 + mov r5, 5 + call 63 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + vm.execute_program_jit().unwrap(); + } +} + +#[test] +fn test_jit_div64_by_zero_imm() { + let prog = assemble( + " + mov32 r0, 1 + div r0, 0 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x0); + } +} + +#[test] +fn test_jit_div_by_zero_imm() { + let prog = assemble( + " + mov32 r0, 1 + div32 r0, 0 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x0); + } +} + +#[test] +fn test_jit_mod64_by_zero_imm() { + let prog = assemble( + " + mov32 r0, 1 + mod r0, 0 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x1); + } +} + +#[test] +fn test_jit_mod_by_zero_imm() { + let prog = assemble( + " + mov32 r0, 1 + mod32 r0, 0 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x1); + } +} + +#[test] +fn test_jit_div64_by_zero_reg() { + let prog = assemble( + " + mov32 r0, 1 + mov32 r1, 0 + div r0, r1 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x0); + } +} + +#[test] +fn test_jit_div_by_zero_reg() { + let prog = assemble( + " + mov32 r0, 1 + mov32 r1, 0 + div32 r0, r1 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x0); + } +} + +#[test] +fn test_jit_mod64_by_zero_reg() { + let prog = assemble( + " + mov32 r0, 1 + mov32 r1, 0 + mod r0, r1 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x1); + } +} + +#[test] +fn test_jit_mod_by_zero_reg() { + let prog = assemble( + " + mov32 r0, 1 + mov32 r1, 0 + mod32 r0, r1 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x1); + } +} + +// TODO SKIP: JIT disabled for this testcase (stack oob check not implemented) +// #[test] +// #[should_panic(expected = "Error: out of bounds memory store (insn #1)")] +// fn test_jit_err_stack_out_of_bound() { +// let prog = &[ +// 0x72, 0x0a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +// 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 +// ]; +// let mut vm = rbpf::EbpfVmNoData::new(Some(prog)).unwrap(); +// vm.jit_compile().unwrap(); +// unsafe { vm.execute_program_jit().unwrap(); } +// } + +#[test] +fn test_jit_exit() { + let prog = assemble( + " + mov r0, 0 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x0); + } +} + +#[test] +fn test_jit_ja() { + let prog = assemble( + " + mov r0, 1 + ja +1 + mov r0, 2 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x1); + } +} + +#[test] +fn test_jit_jeq_imm() { + let prog = assemble( + " + mov32 r0, 0 + mov32 r1, 0xa + jeq r1, 0xb, +4 + mov32 r0, 1 + mov32 r1, 0xb + jeq r1, 0xb, +1 + mov32 r0, 2 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x1); + } +} + +#[test] +fn test_jit_jeq_reg() { + let prog = assemble( + " + mov32 r0, 0 + mov32 r1, 0xa + mov32 r2, 0xb + jeq r1, r2, +4 + mov32 r0, 1 + mov32 r1, 0xb + jeq r1, r2, +1 + mov32 r0, 2 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x1); + } +} + +#[test] +fn test_jit_jge_imm() { + let prog = assemble( + " + mov32 r0, 0 + mov32 r1, 0xa + jge r1, 0xb, +4 + mov32 r0, 1 + mov32 r1, 0xc + jge r1, 0xb, +1 + mov32 r0, 2 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x1); + } +} + +#[test] +fn test_jit_jle_imm() { + let prog = assemble( + " + mov32 r0, 0 + mov32 r1, 5 + jle r1, 4, +1 + jle r1, 6, +1 + exit + jle r1, 5, +1 + exit + mov32 r0, 1 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x1); + } +} + +#[test] +fn test_jit_jle_reg() { + let prog = assemble( + " + mov r0, 0 + mov r1, 5 + mov r2, 4 + mov r3, 6 + jle r1, r2, +2 + jle r1, r1, +1 + exit + jle r1, r3, +1 + exit + mov r0, 1 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x1); + } +} + +#[test] +fn test_jit_jgt_imm() { + let prog = assemble( + " + mov32 r0, 0 + mov32 r1, 5 + jgt r1, 6, +2 + jgt r1, 5, +1 + jgt r1, 4, +1 + exit + mov32 r0, 1 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x1); + } +} + +#[test] +fn test_jit_jgt_reg() { + let prog = assemble( + " + mov r0, 0 + mov r1, 5 + mov r2, 6 + mov r3, 4 + jgt r1, r2, +2 + jgt r1, r1, +1 + jgt r1, r3, +1 + exit + mov r0, 1 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x1); + } +} + +#[test] +fn test_jit_jlt_imm() { + let prog = assemble( + " + mov32 r0, 0 + mov32 r1, 5 + jlt r1, 4, +2 + jlt r1, 5, +1 + jlt r1, 6, +1 + exit + mov32 r0, 1 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x1); + } +} + +#[test] +fn test_jit_jlt_reg() { + let prog = assemble( + " + mov r0, 0 + mov r1, 5 + mov r2, 4 + mov r3, 6 + jlt r1, r2, +2 + jlt r1, r1, +1 + jlt r1, r3, +1 + exit + mov r0, 1 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x1); + } +} + +#[test] +fn test_jit_jit_bounce() { + let prog = assemble( + " + mov r0, 1 + mov r6, r0 + mov r7, r6 + mov r8, r7 + mov r9, r8 + mov r0, r9 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x1); + } +} + +#[test] +fn test_jit_jne_reg() { + let prog = assemble( + " + mov32 r0, 0 + mov32 r1, 0xb + mov32 r2, 0xb + jne r1, r2, +4 + mov32 r0, 1 + mov32 r1, 0xa + jne r1, r2, +1 + mov32 r0, 2 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x1); + } +} + +#[test] +fn test_jit_jset_imm() { + let prog = assemble( + " + mov32 r0, 0 + mov32 r1, 0x7 + jset r1, 0x8, +4 + mov32 r0, 1 + mov32 r1, 0x9 + jset r1, 0x8, +1 + mov32 r0, 2 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x1); + } +} + +#[test] +fn test_jit_jset_reg() { + let prog = assemble( + " + mov32 r0, 0 + mov32 r1, 0x7 + mov32 r2, 0x8 + jset r1, r2, +4 + mov32 r0, 1 + mov32 r1, 0x9 + jset r1, r2, +1 + mov32 r0, 2 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x1); + } +} + +#[test] +fn test_jit_jsge_imm() { + let prog = assemble( + " + mov32 r0, 0 + mov r1, -2 + jsge r1, -1, +5 + jsge r1, 0, +4 + mov32 r0, 1 + mov r1, -1 + jsge r1, -1, +1 + mov32 r0, 2 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x1); + } +} + +#[test] +fn test_jit_jsge_reg() { + let prog = assemble( + " + mov32 r0, 0 + mov r1, -2 + mov r2, -1 + mov32 r3, 0 + jsge r1, r2, +5 + jsge r1, r3, +4 + mov32 r0, 1 + mov r1, r2 + jsge r1, r2, +1 + mov32 r0, 2 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x1); + } +} + +#[test] +fn test_jit_jsle_imm() { + let prog = assemble( + " + mov32 r0, 0 + mov r1, -2 + jsle r1, -3, +1 + jsle r1, -1, +1 + exit + mov32 r0, 1 + jsle r1, -2, +1 + mov32 r0, 2 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x1); + } +} + +#[test] +fn test_jit_jsle_reg() { + let prog = assemble( + " + mov32 r0, 0 + mov r1, -1 + mov r2, -2 + mov32 r3, 0 + jsle r1, r2, +1 + jsle r1, r3, +1 + exit + mov32 r0, 1 + mov r1, r2 + jsle r1, r2, +1 + mov32 r0, 2 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x1); + } +} + +#[test] +fn test_jit_jsgt_imm() { + let prog = assemble( + " + mov32 r0, 0 + mov r1, -2 + jsgt r1, -1, +4 + mov32 r0, 1 + mov32 r1, 0 + jsgt r1, -1, +1 + mov32 r0, 2 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x1); + } +} + +#[test] +fn test_jit_jsgt_reg() { + let prog = assemble( + " + mov32 r0, 0 + mov r1, -2 + mov r2, -1 + jsgt r1, r2, +4 + mov32 r0, 1 + mov32 r1, 0 + jsgt r1, r2, +1 + mov32 r0, 2 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x1); + } +} + +#[test] +fn test_jit_jslt_imm() { + let prog = assemble( + " + mov32 r0, 0 + mov r1, -2 + jslt r1, -3, +2 + jslt r1, -2, +1 + jslt r1, -1, +1 + exit + mov32 r0, 1 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x1); + } +} + +#[test] +fn test_jit_jslt_reg() { + let prog = assemble( + " + mov32 r0, 0 + mov r1, -2 + mov r2, -3 + mov r3, -1 + jslt r1, r1, +2 + jslt r1, r2, +1 + jslt r1, r3, +1 + exit + mov32 r0, 1 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x1); + } +} + +#[test] +fn test_jit_jeq32_imm() { + let prog = assemble( + " + mov r9, 1 + lsh r9, 32 + mov32 r0, 0x0 + mov32 r1, 0xa + jeq32 r1, 0xb, +5 + mov32 r0, 1 + mov r1, 0xb + or r1, r9 + jeq32 r1, 0xb, +1 + mov32 r0, 2 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x1); + } +} + +#[test] +fn test_jit_jeq32_reg() { + let prog = assemble( + " + mov r9, 1 + lsh r9, 32 + mov32 r0, 0 + mov32 r1, 0xa + mov32 r2, 0xb + jeq32 r1, r2, +5 + mov32 r0, 1 + mov32 r1, 0xb + or r1, r9 + jeq32 r1, r2, +1 + mov32 r0, 2 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x1); + } +} + +#[test] +fn test_jit_jge32_imm() { + let prog = assemble( + " + mov r9, 1 + lsh r9, 32 + mov32 r0, 0 + mov32 r1, 0xa + jge32 r1, 0xb, +5 + mov32 r0, 1 + or r1, r9 + mov32 r1, 0xc + jge32 r1, 0xb, +1 + mov32 r0, 2 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x1); + } +} + +#[test] +fn test_jit_jge32_reg() { + let prog = assemble( + " + mov r9, 1 + lsh r9, 32 + mov32 r0, 0 + mov32 r1, 0xa + mov32 r2, 0xb + jge32 r1, r2, +5 + mov32 r0, 1 + or r1, r9 + mov32 r1, 0xc + jge32 r1, r2, +1 + mov32 r0, 2 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x1); + } +} + +#[test] +fn test_jit_jgt32_imm() { + let prog = assemble( + " + mov r9, 1 + lsh r9, 32 + mov32 r0, 0 + mov32 r1, 5 + or r1, r9 + jgt32 r1, 6, +4 + jgt32 r1, 5, +3 + jgt32 r1, 4, +1 + exit + mov32 r0, 1 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x1); + } +} + +#[test] +fn test_jit_jgt32_reg() { + let prog = assemble( + " + mov r9, 1 + lsh r9, 32 + mov r0, 0 + mov r1, 5 + mov32 r1, 5 + or r1, r9 + mov r2, 6 + mov r3, 4 + jgt32 r1, r2, +4 + jgt32 r1, r1, +3 + jgt32 r1, r3, +1 + exit + mov r0, 1 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x1); + } +} + +#[test] +fn test_jit_jle32_imm() { + let prog = assemble( + " + mov r9, 1 + lsh r9, 32 + mov32 r0, 0 + mov32 r1, 5 + or r1, r9 + jle32 r1, 4, +5 + jle32 r1, 6, +1 + exit + jle32 r1, 5, +1 + exit + mov32 r0, 1 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x1); + } +} + +#[test] +fn test_jit_jle32_reg() { + let prog = assemble( + " + mov r9, 1 + lsh r9, 32 + mov r0, 0 + mov r1, 5 + mov r2, 4 + mov r3, 6 + or r1, r9 + jle32 r1, r2, +5 + jle32 r1, r1, +1 + exit + jle32 r1, r3, +1 + exit + mov r0, 1 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x1); + } +} + +#[test] +fn test_jit_jlt32_imm() { + let prog = assemble( + " + mov r9, 1 + lsh r9, 32 + mov32 r0, 0 + mov32 r1, 5 + or r1, r9 + jlt32 r1, 4, +4 + jlt32 r1, 5, +3 + jlt32 r1, 6, +1 + exit + mov32 r0, 1 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x1); + } +} + +#[test] +fn test_jit_jlt32_reg() { + let prog = assemble( + " + mov r9, 1 + lsh r9, 32 + mov r0, 0 + mov r1, 5 + mov r2, 4 + mov r3, 6 + or r1, r9 + jlt32 r1, r2, +4 + jlt32 r1, r1, +3 + jlt32 r1, r3, +1 + exit + mov r0, 1 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x1); + } +} + +#[test] +fn test_jit_jne32_imm() { + let prog = assemble( + " + mov r9, 1 + lsh r9, 32 + mov32 r0, 0 + mov32 r1, 0xb + or r1, r9 + jne32 r1, 0xb, +4 + mov32 r0, 1 + mov32 r1, 0xa + or r1, r9 + jne32 r1, 0xb, +1 + mov32 r0, 2 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x1); + } +} + +#[test] +fn test_jit_jne32_reg() { + let prog = assemble( + " + mov r9, 1 + lsh r9, 32 + mov32 r0, 0 + mov32 r1, 0xb + or r1, r9 + mov32 r2, 0xb + jne32 r1, r2, +4 + mov32 r0, 1 + mov32 r1, 0xa + or r1, r9 + jne32 r1, r2, +1 + mov32 r0, 2 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x1); + } +} + +#[test] +fn test_jit_jset32_imm() { + let prog = assemble( + " + mov r9, 1 + lsh r9, 32 + mov32 r0, 0 + mov32 r1, 0x7 + or r1, r9 + jset32 r1, 0x8, +4 + mov32 r0, 1 + mov32 r1, 0x9 + jset32 r1, 0x8, +1 + mov32 r0, 2 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x1); + } +} + +#[test] +fn test_jit_jset32_reg() { + let prog = assemble( + " + mov r9, 1 + lsh r9, 32 + mov32 r0, 0 + mov32 r1, 0x7 + or r1, r9 + mov32 r2, 0x8 + jset32 r1, r2, +4 + mov32 r0, 1 + mov32 r1, 0x9 + jset32 r1, r2, +1 + mov32 r0, 2 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x1); + } +} + +#[test] +fn test_jit_jsge32_imm() { + let prog = assemble( + " + mov r9, 1 + lsh r9, 32 + mov32 r0, 0 + mov32 r1, -2 + or r1, r9 + jsge32 r1, -1, +5 + jsge32 r1, 0, +4 + mov32 r0, 1 + mov r1, -1 + jsge32 r1, -1, +1 + mov32 r0, 2 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x1); + } +} + +#[test] +fn test_jit_jsge32_reg() { + let prog = assemble( + " + mov r9, 1 + lsh r9, 32 + mov32 r0, 0 + mov32 r1, -2 + or r1, r9 + mov r2, -1 + mov32 r3, 0 + jsge32 r1, r2, +5 + jsge32 r1, r3, +4 + mov32 r0, 1 + mov r1, r2 + jsge32 r1, r2, +1 + mov32 r0, 2 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x1); + } +} + +#[test] +fn test_jit_jsgt32_imm() { + let prog = assemble( + " + mov r9, 1 + lsh r9, 32 + mov32 r0, 0 + mov32 r1, -2 + or r1, r9 + jsgt32 r1, -1, +4 + mov32 r0, 1 + mov32 r1, 0 + jsgt32 r1, -1, +1 + mov32 r0, 2 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x1); + } +} + +#[test] +fn test_jit_jsgt32_reg() { + let prog = assemble( + " + mov r9, 1 + lsh r9, 32 + mov32 r0, 0 + mov32 r1, -2 + or r1, r9 + mov r2, -1 + jsgt32 r1, r2, +4 + mov32 r0, 1 + mov32 r1, 0 + jsgt32 r1, r2, +1 + mov32 r0, 2 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x1); + } +} + +#[test] +fn test_jit_jsle32_imm() { + let prog = assemble( + " + mov r9, 1 + lsh r9, 32 + mov32 r0, 0 + mov32 r1, -2 + or r1, r9 + jsle32 r1, -3, +5 + jsle32 r1, -1, +1 + exit + mov32 r0, 1 + jsle32 r1, -2, +1 + mov32 r0, 2 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x1); + } +} + +#[test] +fn test_jit_jsle32_reg() { + let prog = assemble( + " + mov r9, 1 + lsh r9, 32 + mov32 r0, 0 + mov32 r1, -2 + or r1, r9 + mov r2, -3 + mov32 r3, 0 + jsle32 r1, r2, +6 + jsle32 r1, r3, +1 + exit + mov32 r0, 1 + mov r1, r2 + jsle32 r1, r2, +1 + mov32 r0, 2 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x1); + } +} + +#[test] +fn test_jit_jslt32_imm() { + let prog = assemble( + " + mov r9, 1 + lsh r9, 32 + mov32 r0, 0 + mov32 r1, -2 + or r1, r9 + jslt32 r1, -3, +4 + jslt32 r1, -2, +3 + jslt32 r1, -1, +1 + exit + mov32 r0, 1 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x1); + } +} + +#[test] +fn test_jit_jslt32_reg() { + let prog = assemble( + " + mov r9, 1 + lsh r9, 32 + mov32 r0, 0 + mov32 r1, -2 + or r1, r9 + mov r2, -3 + mov r3, -1 + jslt32 r1, r1, +4 + jslt32 r1, r2, +3 + jslt32 r1, r3, +1 + exit + mov32 r0, 1 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x1); + } +} + +#[test] +fn test_jit_lddw() { + let prog = assemble( + " + lddw r0, 0x1122334455667788 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x1122334455667788); + } +} + +#[test] +fn test_jit_lddw2() { + let prog = assemble( + " + lddw r0, 0x0000000080000000 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x80000000); + } +} + +#[test] +fn test_jit_ldxb_all() { + let prog = assemble( + " + mov r0, r1 + ldxb r9, [r0+0] + lsh r9, 0 + ldxb r8, [r0+1] + lsh r8, 4 + ldxb r7, [r0+2] + lsh r7, 8 + ldxb r6, [r0+3] + lsh r6, 12 + ldxb r5, [r0+4] + lsh r5, 16 + ldxb r4, [r0+5] + lsh r4, 20 + ldxb r3, [r0+6] + lsh r3, 24 + ldxb r2, [r0+7] + lsh r2, 28 + ldxb r1, [r0+8] + lsh r1, 32 + ldxb r0, [r0+9] + lsh r0, 36 + or r0, r1 + or r0, r2 + or r0, r3 + or r0, r4 + or r0, r5 + or r0, r6 + or r0, r7 + or r0, r8 + or r0, r9 + exit", + ) + .unwrap(); + let mem = &mut [0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09]; + let mut vm = rbpf::EbpfVmRaw::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit(mem).unwrap(), 0x9876543210); + } +} + +#[test] +fn test_jit_ldxb() { + let prog = assemble( + " + ldxb r0, [r1+2] + exit", + ) + .unwrap(); + let mem = &mut [0xaa, 0xbb, 0x11, 0xcc, 0xdd]; + let mut vm = rbpf::EbpfVmRaw::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit(mem).unwrap(), 0x11); + } +} + +#[test] +fn test_jit_ldxdw() { + let prog = assemble( + " + ldxdw r0, [r1+2] + exit", + ) + .unwrap(); + let mem = &mut [ + 0xaa, 0xbb, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88, 0xcc, 0xdd, + ]; + let mut vm = rbpf::EbpfVmRaw::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit(mem).unwrap(), 0x8877665544332211); + } +} + +#[test] +fn test_jit_ldxh_all() { + let prog = assemble( + " + mov r0, r1 + ldxh r9, [r0+0] + be16 r9 + lsh r9, 0 + ldxh r8, [r0+2] + be16 r8 + lsh r8, 4 + ldxh r7, [r0+4] + be16 r7 + lsh r7, 8 + ldxh r6, [r0+6] + be16 r6 + lsh r6, 12 + ldxh r5, [r0+8] + be16 r5 + lsh r5, 16 + ldxh r4, [r0+10] + be16 r4 + lsh r4, 20 + ldxh r3, [r0+12] + be16 r3 + lsh r3, 24 + ldxh r2, [r0+14] + be16 r2 + lsh r2, 28 + ldxh r1, [r0+16] + be16 r1 + lsh r1, 32 + ldxh r0, [r0+18] + be16 r0 + lsh r0, 36 + or r0, r1 + or r0, r2 + or r0, r3 + or r0, r4 + or r0, r5 + or r0, r6 + or r0, r7 + or r0, r8 + or r0, r9 + exit", + ) + .unwrap(); + let mem = &mut [ + 0x00, 0x00, 0x00, 0x01, 0x00, 0x02, 0x00, 0x03, 0x00, 0x04, 0x00, 0x05, 0x00, 0x06, 0x00, + 0x07, 0x00, 0x08, 0x00, 0x09, + ]; + let mut vm = rbpf::EbpfVmRaw::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit(mem).unwrap(), 0x9876543210); + } +} + +#[test] +fn test_jit_ldxh_all2() { + let prog = assemble( + " + mov r0, r1 + ldxh r9, [r0+0] + be16 r9 + ldxh r8, [r0+2] + be16 r8 + ldxh r7, [r0+4] + be16 r7 + ldxh r6, [r0+6] + be16 r6 + ldxh r5, [r0+8] + be16 r5 + ldxh r4, [r0+10] + be16 r4 + ldxh r3, [r0+12] + be16 r3 + ldxh r2, [r0+14] + be16 r2 + ldxh r1, [r0+16] + be16 r1 + ldxh r0, [r0+18] + be16 r0 + or r0, r1 + or r0, r2 + or r0, r3 + or r0, r4 + or r0, r5 + or r0, r6 + or r0, r7 + or r0, r8 + or r0, r9 + exit", + ) + .unwrap(); + let mem = &mut [ + 0x00, 0x01, 0x00, 0x02, 0x00, 0x04, 0x00, 0x08, 0x00, 0x10, 0x00, 0x20, 0x00, 0x40, 0x00, + 0x80, 0x01, 0x00, 0x02, 0x00, + ]; + let mut vm = rbpf::EbpfVmRaw::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit(mem).unwrap(), 0x3ff); + } +} + +#[test] +fn test_jit_ldxh() { + let prog = assemble( + " + ldxh r0, [r1+2] + exit", + ) + .unwrap(); + let mem = &mut [0xaa, 0xbb, 0x11, 0x22, 0xcc, 0xdd]; + let mut vm = rbpf::EbpfVmRaw::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit(mem).unwrap(), 0x2211); + } +} + +#[test] +fn test_jit_ldxh_same_reg() { + let prog = assemble( + " + mov r0, r1 + sth [r0], 0x1234 + ldxh r0, [r0] + exit", + ) + .unwrap(); + let mem = &mut [0xff, 0xff]; + let mut vm = rbpf::EbpfVmRaw::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit(mem).unwrap(), 0x1234); + } +} + +#[test] +fn test_jit_ldxw_all() { + let prog = assemble( + " + mov r0, r1 + ldxw r9, [r0+0] + be32 r9 + ldxw r8, [r0+4] + be32 r8 + ldxw r7, [r0+8] + be32 r7 + ldxw r6, [r0+12] + be32 r6 + ldxw r5, [r0+16] + be32 r5 + ldxw r4, [r0+20] + be32 r4 + ldxw r3, [r0+24] + be32 r3 + ldxw r2, [r0+28] + be32 r2 + ldxw r1, [r0+32] + be32 r1 + ldxw r0, [r0+36] + be32 r0 + or r0, r1 + or r0, r2 + or r0, r3 + or r0, r4 + or r0, r5 + or r0, r6 + or r0, r7 + or r0, r8 + or r0, r9 + exit", + ) + .unwrap(); + let mem = &mut [ + 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x08, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x08, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, + ]; + let mut vm = rbpf::EbpfVmRaw::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit(mem).unwrap(), 0x030f0f); + } +} + +#[test] +fn test_jit_ldxw() { + let prog = assemble( + " + ldxw r0, [r1+2] + exit", + ) + .unwrap(); + let mem = &mut [0xaa, 0xbb, 0x11, 0x22, 0x33, 0x44, 0xcc, 0xdd]; + let mut vm = rbpf::EbpfVmRaw::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit(mem).unwrap(), 0x44332211); + } +} + +#[test] +fn test_jit_le16() { + let prog = assemble( + " + ldxh r0, [r1] + le16 r0 + exit", + ) + .unwrap(); + let mem = &mut [0x22, 0x11]; + let mut vm = rbpf::EbpfVmRaw::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit(mem).unwrap(), 0x1122); + } +} + +#[test] +fn test_jit_le32() { + let prog = assemble( + " + ldxw r0, [r1] + le32 r0 + exit", + ) + .unwrap(); + let mem = &mut [0x44, 0x33, 0x22, 0x11]; + let mut vm = rbpf::EbpfVmRaw::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit(mem).unwrap(), 0x11223344); + } +} + +#[test] +fn test_jit_le64() { + let prog = assemble( + " + ldxdw r0, [r1] + le64 r0 + exit", + ) + .unwrap(); + let mem = &mut [0x88, 0x77, 0x66, 0x55, 0x44, 0x33, 0x22, 0x11]; + let mut vm = rbpf::EbpfVmRaw::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit(mem).unwrap(), 0x1122334455667788); + } +} + +#[test] +fn test_jit_lsh_reg() { + let prog = assemble( + " + mov r0, 0x1 + mov r7, 4 + lsh r0, r7 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x10); + } +} + +#[test] +fn test_jit_mod() { + let prog = assemble( + " + mov32 r0, 5748 + mod32 r0, 92 + mov32 r1, 13 + mod32 r0, r1 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x5); + } +} + +#[test] +fn test_jit_mod32() { + let prog = assemble( + " + lddw r0, 0x100000003 + mod32 r0, 3 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x0); + } +} + +#[test] +fn test_jit_mod64() { + let prog = assemble( + " + mov32 r0, -1316649930 + lsh r0, 32 + or r0, 0x100dc5c8 + mov32 r1, 0xdde263e + lsh r1, 32 + or r1, 0x3cbef7f3 + mod r0, r1 + mod r0, 0x658f1778 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x30ba5a04); + } +} + +#[test] +fn test_jit_mov() { + let prog = assemble( + " + mov32 r1, 1 + mov32 r0, r1 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x1); + } +} + +#[test] +fn test_jit_mul32_imm() { + let prog = assemble( + " + mov r0, 3 + mul32 r0, 4 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0xc); + } +} + +#[test] +fn test_jit_mul32_reg() { + let prog = assemble( + " + mov r0, 3 + mov r1, 4 + mul32 r0, r1 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0xc); + } +} + +#[test] +fn test_jit_mul32_reg_overflow() { + let prog = assemble( + " + mov r0, 0x40000001 + mov r1, 4 + mul32 r0, r1 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x4); + } +} + +#[test] +fn test_jit_mul64_imm() { + let prog = assemble( + " + mov r0, 0x40000001 + mul r0, 4 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x100000004); + } +} + +#[test] +fn test_jit_mul64_reg() { + let prog = assemble( + " + mov r0, 0x40000001 + mov r1, 4 + mul r0, r1 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x100000004); + } +} + +#[test] +fn test_jit_mul_loop() { + let prog = assemble( + " + mov r0, 0x7 + add r1, 0xa + lsh r1, 0x20 + rsh r1, 0x20 + jeq r1, 0x0, +4 + mov r0, 0x7 + mul r0, 0x7 + add r1, -1 + jne r1, 0x0, -3 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x75db9c97); + } +} + +#[test] +fn test_jit_neg64() { + let prog = assemble( + " + mov32 r0, 2 + neg r0 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0xfffffffffffffffe); + } +} + +#[test] +fn test_jit_neg() { + let prog = assemble( + " + mov32 r0, 2 + neg32 r0 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0xfffffffe); + } +} + +#[test] +fn test_jit_prime() { + let prog = assemble( + " + mov r1, 67 + mov r0, 0x1 + mov r2, 0x2 + jgt r1, 0x2, +4 + ja +10 + add r2, 0x1 + mov r0, 0x1 + jge r2, r1, +7 + mov r3, r1 + div r3, r2 + mul r3, r2 + mov r4, r1 + sub r4, r3 + mov r0, 0x0 + jne r4, 0x0, -10 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x1); + } +} + +#[test] +fn test_jit_rhs32() { + let prog = assemble( + " + xor r0, r0 + sub r0, 1 + rsh32 r0, 8 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x00ffffff); + } +} + +#[test] +fn test_jit_rsh_reg() { + let prog = assemble( + " + mov r0, 0x10 + mov r7, 4 + rsh r0, r7 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x1); + } +} + +#[test] +fn test_jit_stack() { + let prog = assemble( + " + mov r1, 51 + stdw [r10-16], 0xab + stdw [r10-8], 0xcd + and r1, 1 + lsh r1, 3 + mov r2, r10 + add r2, r1 + ldxdw r0, [r2-16] + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0xcd); + } +} + +#[test] +fn test_jit_stack2() { + let prog = assemble( + " + stb [r10-4], 0x01 + stb [r10-3], 0x02 + stb [r10-2], 0x03 + stb [r10-1], 0x04 + mov r1, r10 + mov r2, 0x4 + sub r1, r2 + call 1 + mov r1, 0 + ldxb r2, [r10-4] + ldxb r3, [r10-3] + ldxb r4, [r10-2] + ldxb r5, [r10-1] + call 0 + xor r0, 0x2a2a2a2a + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.register_helper(0, helpers::gather_bytes).unwrap(); + vm.register_helper(1, helpers::memfrob).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x01020304); + } +} + +#[test] +fn test_jit_stb() { + let prog = assemble( + " + stb [r1+2], 0x11 + ldxb r0, [r1+2] + exit", + ) + .unwrap(); + let mem = &mut [0xaa, 0xbb, 0xff, 0xcc, 0xdd]; + let mut vm = rbpf::EbpfVmRaw::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit(mem).unwrap(), 0x11); + } +} + +#[test] +fn test_jit_stdw() { + let prog = assemble( + " + stdw [r1+2], 0x44332211 + ldxdw r0, [r1+2] + exit", + ) + .unwrap(); + let mem = &mut [ + 0xaa, 0xbb, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xcc, 0xdd, + ]; + let mut vm = rbpf::EbpfVmRaw::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit(mem).unwrap(), 0x44332211); + } +} + +#[test] +fn test_jit_sth() { + let prog = assemble( + " + sth [r1+2], 0x2211 + ldxh r0, [r1+2] + exit", + ) + .unwrap(); + let mem = &mut [0xaa, 0xbb, 0xff, 0xff, 0xcc, 0xdd]; + let mut vm = rbpf::EbpfVmRaw::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit(mem).unwrap(), 0x2211); + } +} + +#[test] +fn test_jit_string_stack() { + let prog = assemble( + " + mov r1, 0x78636261 + stxw [r10-8], r1 + mov r6, 0x0 + stxb [r10-4], r6 + stxb [r10-12], r6 + mov r1, 0x79636261 + stxw [r10-16], r1 + mov r1, r10 + add r1, -8 + mov r2, r1 + call 0x4 + mov r1, r0 + mov r0, 0x1 + lsh r1, 0x20 + rsh r1, 0x20 + jne r1, 0x0, +11 + mov r1, r10 + add r1, -8 + mov r2, r10 + add r2, -16 + call 0x4 + mov r1, r0 + lsh r1, 0x20 + rsh r1, 0x20 + mov r0, 0x1 + jeq r1, r6, +1 + mov r0, 0x0 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.register_helper(4, helpers::strcmp).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x0); + } +} + +#[test] +fn test_jit_stw() { + let prog = assemble( + " + stw [r1+2], 0x44332211 + ldxw r0, [r1+2] + exit", + ) + .unwrap(); + let mem = &mut [0xaa, 0xbb, 0xff, 0xff, 0xff, 0xff, 0xcc, 0xdd]; + let mut vm = rbpf::EbpfVmRaw::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit(mem).unwrap(), 0x44332211); + } +} + +#[test] +fn test_jit_stxb() { + let prog = assemble( + " + mov32 r2, 0x11 + stxb [r1+2], r2 + ldxb r0, [r1+2] + exit", + ) + .unwrap(); + let mem = &mut [0xaa, 0xbb, 0xff, 0xcc, 0xdd]; + let mut vm = rbpf::EbpfVmRaw::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit(mem).unwrap(), 0x11); + } +} + +#[test] +fn test_jit_stxb_all() { + let prog = assemble( + " + mov r0, 0xf0 + mov r2, 0xf2 + mov r3, 0xf3 + mov r4, 0xf4 + mov r5, 0xf5 + mov r6, 0xf6 + mov r7, 0xf7 + mov r8, 0xf8 + stxb [r1], r0 + stxb [r1+1], r2 + stxb [r1+2], r3 + stxb [r1+3], r4 + stxb [r1+4], r5 + stxb [r1+5], r6 + stxb [r1+6], r7 + stxb [r1+7], r8 + ldxdw r0, [r1] + be64 r0 + exit", + ) + .unwrap(); + let mem = &mut [0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff]; + let mut vm = rbpf::EbpfVmRaw::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit(mem).unwrap(), 0xf0f2f3f4f5f6f7f8); + } +} + +#[test] +fn test_jit_stxb_all2() { + let prog = assemble( + " + mov r0, r1 + mov r1, 0xf1 + mov r9, 0xf9 + stxb [r0], r1 + stxb [r0+1], r9 + ldxh r0, [r0] + be16 r0 + exit", + ) + .unwrap(); + let mem = &mut [0xff, 0xff]; + let mut vm = rbpf::EbpfVmRaw::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit(mem).unwrap(), 0xf1f9); + } +} + +#[test] +fn test_jit_stxb_chain() { + let prog = assemble( + " + mov r0, r1 + ldxb r9, [r0+0] + stxb [r0+1], r9 + ldxb r8, [r0+1] + stxb [r0+2], r8 + ldxb r7, [r0+2] + stxb [r0+3], r7 + ldxb r6, [r0+3] + stxb [r0+4], r6 + ldxb r5, [r0+4] + stxb [r0+5], r5 + ldxb r4, [r0+5] + stxb [r0+6], r4 + ldxb r3, [r0+6] + stxb [r0+7], r3 + ldxb r2, [r0+7] + stxb [r0+8], r2 + ldxb r1, [r0+8] + stxb [r0+9], r1 + ldxb r0, [r0+9] + exit", + ) + .unwrap(); + let mem = &mut [0x2a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00]; + let mut vm = rbpf::EbpfVmRaw::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit(mem).unwrap(), 0x2a); + } +} + +#[test] +fn test_jit_stxdw() { + let prog = assemble( + " + mov r2, -2005440939 + lsh r2, 32 + or r2, 0x44332211 + stxdw [r1+2], r2 + ldxdw r0, [r1+2] + exit", + ) + .unwrap(); + let mem = &mut [ + 0xaa, 0xbb, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xcc, 0xdd, + ]; + let mut vm = rbpf::EbpfVmRaw::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit(mem).unwrap(), 0x8877665544332211); + } +} + +#[test] +fn test_jit_stxh() { + let prog = assemble( + " + mov32 r2, 0x2211 + stxh [r1+2], r2 + ldxh r0, [r1+2] + exit", + ) + .unwrap(); + let mem = &mut [0xaa, 0xbb, 0xff, 0xff, 0xcc, 0xdd]; + let mut vm = rbpf::EbpfVmRaw::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit(mem).unwrap(), 0x2211); + } +} + +#[test] +fn test_jit_stxw() { + let prog = assemble( + " + mov32 r2, 0x44332211 + stxw [r1+2], r2 + ldxw r0, [r1+2] + exit", + ) + .unwrap(); + let mem = &mut [0xaa, 0xbb, 0xff, 0xff, 0xff, 0xff, 0xcc, 0xdd]; + let mut vm = rbpf::EbpfVmRaw::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit(mem).unwrap(), 0x44332211); + } +} + +#[test] +fn test_jit_subnet() { + let prog = assemble( + " + mov r2, 0xe + ldxh r3, [r1+12] + jne r3, 0x81, +2 + mov r2, 0x12 + ldxh r3, [r1+16] + and r3, 0xffff + jne r3, 0x8, +5 + add r1, r2 + mov r0, 0x1 + ldxw r1, [r1+16] + and r1, 0xffffff + jeq r1, 0x1a8c0, +1 + mov r0, 0x0 + exit", + ) + .unwrap(); + let mem = &mut [ + 0x00, 0x00, 0xc0, 0x9f, 0xa0, 0x97, 0x00, 0xa0, 0xcc, 0x3b, 0xbf, 0xfa, 0x08, 0x00, 0x45, + 0x10, 0x00, 0x3c, 0x46, 0x3c, 0x40, 0x00, 0x40, 0x06, 0x73, 0x1c, 0xc0, 0xa8, 0x01, 0x02, + 0xc0, 0xa8, 0x01, 0x01, 0x06, 0x0e, 0x00, 0x17, 0x99, 0xc5, 0xa0, 0xec, 0x00, 0x00, 0x00, + 0x00, 0xa0, 0x02, 0x7d, 0x78, 0xe0, 0xa3, 0x00, 0x00, 0x02, 0x04, 0x05, 0xb4, 0x04, 0x02, + 0x08, 0x0a, 0x00, 0x9c, 0x27, 0x24, 0x00, 0x00, 0x00, 0x00, 0x01, 0x03, 0x03, 0x00, + ]; + let mut vm = rbpf::EbpfVmRaw::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit(mem).unwrap(), 0x1); + } +} + +const PROG_TCP_PORT_80: [u8; 152] = [ + 0x71, 0x12, 0x0c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x71, 0x13, 0x0d, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x67, 0x03, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x4f, 0x23, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xb7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x55, 0x03, 0x0c, 0x00, 0x08, 0x00, 0x00, 0x00, + 0x71, 0x12, 0x17, 0x00, 0x00, 0x00, 0x00, 0x00, 0x55, 0x02, 0x0a, 0x00, 0x06, 0x00, 0x00, 0x00, + 0x71, 0x12, 0x0e, 0x00, 0x00, 0x00, 0x00, 0x00, 0x07, 0x01, 0x00, 0x00, 0x0e, 0x00, 0x00, 0x00, + 0x57, 0x02, 0x00, 0x00, 0x0f, 0x00, 0x00, 0x00, 0x67, 0x02, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, + 0x0f, 0x21, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x69, 0x12, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x15, 0x02, 0x02, 0x00, 0x00, 0x50, 0x00, 0x00, 0x69, 0x11, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x55, 0x01, 0x01, 0x00, 0x00, 0x50, 0x00, 0x00, 0xb7, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +]; + +#[test] +fn test_jit_tcp_port80_match() { + let mem = &mut [ + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x00, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x08, 0x00, 0x45, + 0x00, 0x00, 0x56, 0x00, 0x01, 0x00, 0x00, 0x40, 0x06, 0xf9, 0x4d, 0xc0, 0xa8, 0x00, 0x01, + 0xc0, 0xa8, 0x00, 0x02, 0x27, 0x10, 0x00, 0x50, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x50, 0x02, 0x20, 0x00, 0xc5, 0x18, 0x00, 0x00, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, + 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, + 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, + 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, + ]; + let prog = &PROG_TCP_PORT_80; + let mut vm = rbpf::EbpfVmRaw::new(Some(prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit(mem).unwrap(), 0x1); + } +} + +#[test] +fn test_jit_tcp_port80_nomatch() { + let mem = &mut [ + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x00, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x08, 0x00, 0x45, + 0x00, 0x00, 0x56, 0x00, 0x01, 0x00, 0x00, 0x40, 0x06, 0xf9, 0x4d, 0xc0, 0xa8, 0x00, 0x01, + 0xc0, 0xa8, 0x00, 0x02, 0x00, 0x16, 0x27, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x51, 0x02, 0x20, 0x00, 0xc5, 0x18, 0x00, 0x00, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, + 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, + 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, + 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, + ]; + let prog = &PROG_TCP_PORT_80; + let mut vm = rbpf::EbpfVmRaw::new(Some(prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit(mem).unwrap(), 0x0); + } +} + +#[test] +fn test_jit_tcp_port80_nomatch_ethertype() { + let mem = &mut [ + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x00, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x08, 0x01, 0x45, + 0x00, 0x00, 0x56, 0x00, 0x01, 0x00, 0x00, 0x40, 0x06, 0xf9, 0x4d, 0xc0, 0xa8, 0x00, 0x01, + 0xc0, 0xa8, 0x00, 0x02, 0x27, 0x10, 0x00, 0x50, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x50, 0x02, 0x20, 0x00, 0xc5, 0x18, 0x00, 0x00, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, + 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, + 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, + 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, + ]; + let prog = &PROG_TCP_PORT_80; + let mut vm = rbpf::EbpfVmRaw::new(Some(prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit(mem).unwrap(), 0x0); + } +} + +#[test] +fn test_jit_tcp_port80_nomatch_proto() { + let mem = &mut [ + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x00, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x08, 0x00, 0x45, + 0x00, 0x00, 0x56, 0x00, 0x01, 0x00, 0x00, 0x40, 0x11, 0xf9, 0x4d, 0xc0, 0xa8, 0x00, 0x01, + 0xc0, 0xa8, 0x00, 0x02, 0x27, 0x10, 0x00, 0x50, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x50, 0x02, 0x20, 0x00, 0xc5, 0x18, 0x00, 0x00, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, + 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, + 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, + 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, + ]; + let prog = &PROG_TCP_PORT_80; + let mut vm = rbpf::EbpfVmRaw::new(Some(prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit(mem).unwrap(), 0x0); + } +} + +#[test] +fn test_jit_tcp_sack_match() { + let mut mem = TCP_SACK_MATCH.to_vec(); + let prog = assemble(TCP_SACK_ASM).unwrap(); + let mut vm = rbpf::EbpfVmRaw::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit(mem.as_mut_slice()).unwrap(), 0x1); + } +} + +#[test] +fn test_jit_tcp_sack_nomatch() { + let mut mem = TCP_SACK_NOMATCH.to_vec(); + let prog = assemble(TCP_SACK_ASM).unwrap(); + let mut vm = rbpf::EbpfVmRaw::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit(mem.as_mut_slice()).unwrap(), 0x0); + } +} diff --git a/kernel/crates/rbpf/tests/ubpf_verifier.rs b/kernel/crates/rbpf/tests/ubpf_verifier.rs new file mode 100644 index 000000000..8e6b03afe --- /dev/null +++ b/kernel/crates/rbpf/tests/ubpf_verifier.rs @@ -0,0 +1,177 @@ +// SPDX-License-Identifier: (Apache-2.0 OR MIT) +// Converted from the tests for uBPF +// Copyright 2015 Big Switch Networks, Inc +// Copyright 2016 6WIND S.A. + +// The tests contained in this file are extracted from the unit tests of uBPF software. Each test +// in this file has a name in the form `test_verifier_`, and corresponds to the +// (human-readable) code in `ubpf/tree/master/tests/`, available at +// (hyphen had to be replaced with underscores +// as Rust will not accept them in function names). It is strongly advised to refer to the uBPF +// version to understand what these program do. +// +// Each program was assembled from the uBPF version with the assembler provided by uBPF itself, and +// available at . +// The very few modifications that have been realized should be indicated. + +// These are unit tests for the eBPF “verifier”. + +extern crate rbpf; + +use rbpf::{assembler::assemble, ebpf}; + +#[test] +#[should_panic(expected = "[Verifier] Error: unsupported argument for LE/BE (insn #0)")] +fn test_verifier_err_endian_size() { + let prog = &[ + 0xdc, 0x01, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0xb7, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + ]; + let vm = rbpf::EbpfVmNoData::new(Some(prog)).unwrap(); + vm.execute_program().unwrap(); +} + +#[test] +#[should_panic(expected = "[Verifier] Error: incomplete LD_DW instruction (insn #0)")] +fn test_verifier_err_incomplete_lddw() { + // Note: ubpf has test-err-incomplete-lddw2, which is the same + let prog = &[ + 0x18, 0x00, 0x00, 0x00, 0x88, 0x77, 0x66, 0x55, 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, + ]; + let vm = rbpf::EbpfVmNoData::new(Some(prog)).unwrap(); + vm.execute_program().unwrap(); +} + +#[test] +#[should_panic(expected = "[Verifier] Error: infinite loop")] +fn test_verifier_err_infinite_loop() { + let prog = assemble( + " + ja -1 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.execute_program().unwrap(); +} + +#[test] +#[should_panic(expected = "[Verifier] Error: invalid destination register (insn #0)")] +fn test_verifier_err_invalid_reg_dst() { + let prog = assemble( + " + mov r11, 1 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.execute_program().unwrap(); +} + +#[test] +#[should_panic(expected = "[Verifier] Error: invalid source register (insn #0)")] +fn test_verifier_err_invalid_reg_src() { + let prog = assemble( + " + mov r0, r11 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.execute_program().unwrap(); +} + +#[test] +#[should_panic(expected = "[Verifier] Error: jump to middle of LD_DW at #2 (insn #0)")] +fn test_verifier_err_jmp_lddw() { + let prog = assemble( + " + ja +1 + lddw r0, 0x1122334455667788 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.execute_program().unwrap(); +} + +#[test] +#[should_panic(expected = "[Verifier] Error: jump out of code to #3 (insn #0)")] +fn test_verifier_err_jmp_out() { + let prog = assemble( + " + ja +2 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.execute_program().unwrap(); +} + +#[test] +#[should_panic(expected = "[Verifier] Error: program does not end with “EXIT” instruction")] +fn test_verifier_err_no_exit() { + let prog = assemble( + " + mov32 r0, 0", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.execute_program().unwrap(); +} + +#[test] +fn test_verifier_err_no_exit_backward_jump() { + let prog = assemble( + " + ja +1 + exit + ja -2", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.execute_program().unwrap(); +} + +#[test] +#[should_panic(expected = "[Verifier] Error: eBPF program length limited to 1000000, here 1000001")] +fn test_verifier_err_too_many_instructions() { + // uBPF uses 65637 instructions, because it sets its limit at 65636. + // We use the classic 4096 limit from kernel, so no need to produce as many instructions. + let mut prog = (0..(1_000_000 * ebpf::INSN_SIZE)) + .map(|x| match x % 8 { + 0 => 0xb7, + 1 => 0x01, + _ => 0, + }) + .collect::>(); + prog.append(&mut vec![0x95, 0, 0, 0, 0, 0, 0, 0]); + + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.execute_program().unwrap(); +} + +#[test] +#[should_panic(expected = "[Verifier] Error: unknown eBPF opcode 0x6 (insn #0)")] +fn test_verifier_err_unknown_opcode() { + let prog = &[ + 0x06, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, + ]; + let vm = rbpf::EbpfVmNoData::new(Some(prog)).unwrap(); + vm.execute_program().unwrap(); +} + +#[test] +#[should_panic(expected = "[Verifier] Error: cannot write into register r10 (insn #0)")] +fn test_verifier_err_write_r10() { + let prog = assemble( + " + mov r10, 1 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.execute_program().unwrap(); +} diff --git a/kernel/crates/rbpf/tests/ubpf_vm.rs b/kernel/crates/rbpf/tests/ubpf_vm.rs new file mode 100644 index 000000000..05d8a9baa --- /dev/null +++ b/kernel/crates/rbpf/tests/ubpf_vm.rs @@ -0,0 +1,2670 @@ +// SPDX-License-Identifier: (Apache-2.0 OR MIT) +// Converted from the tests for uBPF +// Copyright 2015 Big Switch Networks, Inc +// Copyright 2016 6WIND S.A. + +// The tests contained in this file are extracted from the unit tests of uBPF software. Each test +// in this file has a name in the form `test_vm_`, and corresponds to the (human-readable) +// code in `ubpf/tree/master/tests/`, available at +// (hyphen had to be replaced with underscores +// as Rust will not accept them in function names). It is strongly advised to refer to the uBPF +// version to understand what these program do. +// +// Each program was assembled from the uBPF version with the assembler provided by uBPF itself, and +// available at . +// The very few modifications that have been realized should be indicated. + +// These are unit tests for the eBPF interpreter. + +#![allow(clippy::unreadable_literal)] + +extern crate rbpf; +mod common; + +use common::{TCP_SACK_ASM, TCP_SACK_MATCH, TCP_SACK_NOMATCH}; +use rbpf::{assembler::assemble, helpers}; + +#[test] +fn test_vm_add() { + let prog = assemble( + " + mov32 r0, 0 + mov32 r1, 2 + add32 r0, 1 + add32 r0, r1 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x3); +} + +#[test] +fn test_vm_alu64_arith() { + let prog = assemble( + " + mov r0, 0 + mov r1, 1 + mov r2, 2 + mov r3, 3 + mov r4, 4 + mov r5, 5 + mov r6, 6 + mov r7, 7 + mov r8, 8 + mov r9, 9 + add r0, 23 + add r0, r7 + sub r0, 13 + sub r0, r1 + mul r0, 7 + mul r0, r3 + div r0, 2 + div r0, r4 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x2a); +} + +#[test] +fn test_vm_alu64_bit() { + let prog = assemble( + " + mov r0, 0 + mov r1, 1 + mov r2, 2 + mov r3, 3 + mov r4, 4 + mov r5, 5 + mov r6, 6 + mov r7, 7 + mov r8, 8 + or r0, r5 + or r0, 0xa0 + and r0, 0xa3 + mov r9, 0x91 + and r0, r9 + lsh r0, 32 + lsh r0, 22 + lsh r0, r8 + rsh r0, 32 + rsh r0, 19 + rsh r0, r7 + xor r0, 0x03 + xor r0, r2 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x11); +} + +#[test] +fn test_vm_alu_arith() { + let prog = assemble( + " + mov32 r0, 0 + mov32 r1, 1 + mov32 r2, 2 + mov32 r3, 3 + mov32 r4, 4 + mov32 r5, 5 + mov32 r6, 6 + mov32 r7, 7 + mov32 r8, 8 + mov32 r9, 9 + add32 r0, 23 + add32 r0, r7 + sub32 r0, 13 + sub32 r0, r1 + mul32 r0, 7 + mul32 r0, r3 + div32 r0, 2 + div32 r0, r4 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x2a); +} + +#[test] +fn test_vm_alu_bit() { + let prog = assemble( + " + mov32 r0, 0 + mov32 r1, 1 + mov32 r2, 2 + mov32 r3, 3 + mov32 r4, 4 + mov32 r5, 5 + mov32 r6, 6 + mov32 r7, 7 + mov32 r8, 8 + or32 r0, r5 + or32 r0, 0xa0 + and32 r0, 0xa3 + mov32 r9, 0x91 + and32 r0, r9 + lsh32 r0, 22 + lsh32 r0, r8 + rsh32 r0, 19 + rsh32 r0, r7 + xor32 r0, 0x03 + xor32 r0, r2 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x11); +} + +#[test] +fn test_vm_arsh32_high_shift() { + let prog = assemble( + " + mov r0, 8 + lddw r1, 0x100000001 + arsh32 r0, r1 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x4); +} + +#[test] +fn test_vm_arsh() { + let prog = assemble( + " + mov32 r0, 0xf8 + lsh32 r0, 28 + arsh32 r0, 16 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0xffff8000); +} + +#[test] +fn test_vm_arsh64() { + let prog = assemble( + " + mov32 r0, 1 + lsh r0, 63 + arsh r0, 55 + mov32 r1, 5 + arsh r0, r1 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0xfffffffffffffff8); +} + +#[test] +fn test_vm_arsh_reg() { + let prog = assemble( + " + mov32 r0, 0xf8 + mov32 r1, 16 + lsh32 r0, 28 + arsh32 r0, r1 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0xffff8000); +} + +#[test] +fn test_vm_arsh_imm_overflow() { + let prog = assemble( + " + mov r0, 1 + lsh r0, 63 + arsh r0, 0xff20 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0xffffffff80000000); +} + +#[test] +fn test_vm_arsh_reg_overflow() { + let prog = assemble( + " + mov r0, 1 + lsh r0, 63 + mov r1, 0xff04 + arsh r0, r1 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0xf800000000000000); +} + +#[test] +fn test_vm_arsh32_imm_overflow() { + let prog = assemble( + " + mov32 r0, 1 + lsh32 r0, 31 + arsh32 r0, 0xff10 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0xffff8000); +} + +#[test] +fn test_vm_arsh32_reg_overflow() { + let prog = assemble( + " + mov32 r0, 1 + lsh32 r0, 31 + mov32 r1, 32 + arsh32 r0, r1 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x80000000); +} + +#[test] +fn test_vm_be16() { + let prog = assemble( + " + ldxh r0, [r1] + be16 r0 + exit", + ) + .unwrap(); + let mem = &mut [0x11, 0x22]; + let vm = rbpf::EbpfVmRaw::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program(mem).unwrap(), 0x1122); +} + +#[test] +fn test_vm_be16_high() { + let prog = assemble( + " + ldxdw r0, [r1] + be16 r0 + exit", + ) + .unwrap(); + let mem = &mut [0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88]; + let vm = rbpf::EbpfVmRaw::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program(mem).unwrap(), 0x1122); +} + +#[test] +fn test_vm_be32() { + let prog = assemble( + " + ldxw r0, [r1] + be32 r0 + exit", + ) + .unwrap(); + let mem = &mut [0x11, 0x22, 0x33, 0x44]; + let vm = rbpf::EbpfVmRaw::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program(mem).unwrap(), 0x11223344); +} + +#[test] +fn test_vm_be32_high() { + let prog = assemble( + " + ldxdw r0, [r1] + be32 r0 + exit", + ) + .unwrap(); + let mem = &mut [0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88]; + let vm = rbpf::EbpfVmRaw::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program(mem).unwrap(), 0x11223344); +} + +#[test] +fn test_vm_be64() { + let prog = assemble( + " + ldxdw r0, [r1] + be64 r0 + exit", + ) + .unwrap(); + let mem = &mut [0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88]; + let vm = rbpf::EbpfVmRaw::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program(mem).unwrap(), 0x1122334455667788); +} + +#[test] +fn test_vm_call() { + let prog = assemble( + " + mov r1, 1 + mov r2, 2 + mov r3, 3 + mov r4, 4 + mov r5, 5 + call 0 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.register_helper(0, helpers::gather_bytes).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x0102030405); +} + +#[test] +fn test_vm_call_memfrob() { + let prog = assemble( + " + mov r6, r1 + add r1, 2 + mov r2, 4 + call 1 + ldxdw r0, [r6] + be64 r0 + exit", + ) + .unwrap(); + let mem = &mut [0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08]; + let mut vm = rbpf::EbpfVmRaw::new(Some(&prog)).unwrap(); + vm.register_helper(1, helpers::memfrob).unwrap(); + assert_eq!(vm.execute_program(mem).unwrap(), 0x102292e2f2c0708); +} + +// TODO: helpers::trash_registers needs asm!(). +// Try this again once asm!() is available in stable. +//#[test] +//fn test_vm_call_save() { +//let prog = &[ +//0xb7, 0x06, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, +//0xb7, 0x07, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, +//0xb7, 0x08, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, +//0xb7, 0x09, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, +//0x85, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, +//0xb7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +//0x4f, 0x60, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +//0x4f, 0x70, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +//0x4f, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +//0x4f, 0x90, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +//0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 +//]; +//let mut vm = rbpf::EbpfVmNoData::new(Some(prog)).unwrap(); +//vm.register_helper(2, helpers::trash_registers); +//assert_eq!(vm.execute_program().unwrap(), 0x4321); +//} + +#[test] +fn test_vm_div32_high_divisor() { + let prog = assemble( + " + mov r0, 12 + lddw r1, 0x100000004 + div32 r0, r1 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x3); +} + +#[test] +fn test_vm_div32_imm() { + let prog = assemble( + " + lddw r0, 0x10000000c + div32 r0, 4 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x3); +} + +#[test] +fn test_vm_div32_reg() { + let prog = assemble( + " + lddw r0, 0x10000000c + mov r1, 4 + div32 r0, r1 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x3); +} + +#[test] +fn test_vm_div64_imm() { + let prog = assemble( + " + mov r0, 0xc + lsh r0, 32 + div r0, 4 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x300000000); +} + +#[test] +fn test_vm_div64_reg() { + let prog = assemble( + " + mov r0, 0xc + lsh r0, 32 + mov r1, 4 + div r0, r1 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x300000000); +} + +#[test] +fn test_vm_early_exit() { + let prog = assemble( + " + mov r0, 3 + exit + mov r0, 4 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x3); +} + +// uBPF limits the number of user functions at 64. We don't. +//#[test] +//fn test_vm_err_call_bad_imm() { +//} + +#[test] +#[should_panic(expected = "Error: unknown helper function (id: 0x3f)")] +fn test_vm_err_call_unreg() { + let prog = assemble( + " + mov r1, 1 + mov r2, 2 + mov r3, 3 + mov r4, 4 + mov r5, 5 + call 63 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.execute_program().unwrap(); +} + +#[test] +fn test_vm_div64_by_zero_imm() { + let prog = assemble( + " + mov32 r0, 1 + div r0, 0 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x0); +} + +#[test] +fn test_vm_div_by_zero_imm() { + let prog = assemble( + " + mov32 r0, 1 + div32 r0, 0 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x0); +} + +#[test] +fn test_vm_mod64_by_zero_imm() { + let prog = assemble( + " + mov32 r0, 1 + mod r0, 0 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x1); +} + +#[test] +fn test_vm_mod_by_zero_imm() { + let prog = assemble( + " + mov32 r0, 1 + mod32 r0, 0 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x1); +} + +// Make sure we only consider the last 32 bits of the divisor. +#[test] +fn test_vm_mod_by_zero_reg_long() { + let prog = assemble( + " + lddw r1, 0x100000000 + mod32 r0, r1 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x0); +} + +#[test] +fn test_vm_div64_by_zero_reg() { + let prog = assemble( + " + mov32 r0, 1 + mov32 r1, 0 + div r0, r1 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x0); +} + +#[test] +fn test_vm_div_by_zero_reg() { + let prog = assemble( + " + mov32 r0, 1 + mov32 r1, 0 + div32 r0, r1 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x0); +} + +// Make sure we only consider the last 32 bits of the divisor. +#[test] +fn test_vm_div_by_zero_reg_long() { + let prog = assemble( + " + lddw r1, 0x100000000 + div32 r0, r1 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x0); +} + +#[test] +fn test_vm_mod64_by_zero_reg() { + let prog = assemble( + " + mov32 r0, 1 + mov32 r1, 0 + mod r0, r1 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x1); +} + +#[test] +fn test_vm_mod_by_zero_reg() { + let prog = assemble( + " + mov32 r0, 1 + mov32 r1, 0 + mod32 r0, r1 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x1); +} + +#[test] +#[should_panic(expected = "Error: out of bounds memory store (insn #1)")] +fn test_vm_err_stack_out_of_bound() { + let prog = assemble( + " + stb [r10], 0 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.execute_program().unwrap(); +} + +#[test] +fn test_vm_exit() { + let prog = assemble( + " + mov r0, 0 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x0); +} + +#[test] +fn test_vm_ja() { + let prog = assemble( + " + mov r0, 1 + ja +1 + mov r0, 2 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x1); +} + +#[test] +fn test_vm_jeq_imm() { + let prog = assemble( + " + mov32 r0, 0 + mov32 r1, 0xa + jeq r1, 0xb, +4 + mov32 r0, 1 + mov32 r1, 0xb + jeq r1, 0xb, +1 + mov32 r0, 2 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x1); +} + +#[test] +fn test_vm_jeq_reg() { + let prog = assemble( + " + mov32 r0, 0 + mov32 r1, 0xa + mov32 r2, 0xb + jeq r1, r2, +4 + mov32 r0, 1 + mov32 r1, 0xb + jeq r1, r2, +1 + mov32 r0, 2 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x1); +} + +#[test] +fn test_vm_jge_imm() { + let prog = assemble( + " + mov32 r0, 0 + mov32 r1, 0xa + jge r1, 0xb, +4 + mov32 r0, 1 + mov32 r1, 0xc + jge r1, 0xb, +1 + mov32 r0, 2 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x1); +} + +#[test] +fn test_vm_jle_imm() { + let prog = assemble( + " + mov32 r0, 0 + mov32 r1, 5 + jle r1, 4, +1 + jle r1, 6, +1 + exit + jle r1, 5, +1 + exit + mov32 r0, 1 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x1); +} + +#[test] +fn test_vm_jle_reg() { + let prog = assemble( + " + mov r0, 0 + mov r1, 5 + mov r2, 4 + mov r3, 6 + jle r1, r2, +2 + jle r1, r1, +1 + exit + jle r1, r3, +1 + exit + mov r0, 1 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x1); +} + +#[test] +fn test_vm_jgt_imm() { + let prog = assemble( + " + mov32 r0, 0 + mov32 r1, 5 + jgt r1, 6, +2 + jgt r1, 5, +1 + jgt r1, 4, +1 + exit + mov32 r0, 1 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x1); +} + +#[test] +fn test_vm_jgt_reg() { + let prog = assemble( + " + mov r0, 0 + mov r1, 5 + mov r2, 6 + mov r3, 4 + jgt r1, r2, +2 + jgt r1, r1, +1 + jgt r1, r3, +1 + exit + mov r0, 1 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x1); +} + +#[test] +fn test_vm_jlt_imm() { + let prog = assemble( + " + mov32 r0, 0 + mov32 r1, 5 + jlt r1, 4, +2 + jlt r1, 5, +1 + jlt r1, 6, +1 + exit + mov32 r0, 1 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x1); +} + +#[test] +fn test_vm_jlt_reg() { + let prog = assemble( + " + mov r0, 0 + mov r1, 5 + mov r2, 4 + mov r3, 6 + jlt r1, r2, +2 + jlt r1, r1, +1 + jlt r1, r3, +1 + exit + mov r0, 1 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x1); +} + +#[test] +fn test_vm_jit_bounce() { + let prog = assemble( + " + mov r0, 1 + mov r6, r0 + mov r7, r6 + mov r8, r7 + mov r9, r8 + mov r0, r9 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x1); +} + +#[test] +fn test_vm_jne_reg() { + let prog = assemble( + " + mov32 r0, 0 + mov32 r1, 0xb + mov32 r2, 0xb + jne r1, r2, +4 + mov32 r0, 1 + mov32 r1, 0xa + jne r1, r2, +1 + mov32 r0, 2 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x1); +} + +#[test] +fn test_vm_jset_imm() { + let prog = assemble( + " + mov32 r0, 0 + mov32 r1, 0x7 + jset r1, 0x8, +4 + mov32 r0, 1 + mov32 r1, 0x9 + jset r1, 0x8, +1 + mov32 r0, 2 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x1); +} + +#[test] +fn test_vm_jset_reg() { + let prog = assemble( + " + mov32 r0, 0 + mov32 r1, 0x7 + mov32 r2, 0x8 + jset r1, r2, +4 + mov32 r0, 1 + mov32 r1, 0x9 + jset r1, r2, +1 + mov32 r0, 2 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x1); +} + +#[test] +fn test_vm_jsge_imm() { + let prog = assemble( + " + mov32 r0, 0 + mov r1, -2 + jsge r1, -1, +5 + jsge r1, 0, +4 + mov32 r0, 1 + mov r1, -1 + jsge r1, -1, +1 + mov32 r0, 2 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x1); +} + +#[test] +fn test_vm_jsge_reg() { + let prog = assemble( + " + mov32 r0, 0 + mov r1, -2 + mov r2, -1 + mov32 r3, 0 + jsge r1, r2, +5 + jsge r1, r3, +4 + mov32 r0, 1 + mov r1, r2 + jsge r1, r2, +1 + mov32 r0, 2 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x1); +} + +#[test] +fn test_vm_jsle_imm() { + let prog = assemble( + " + mov32 r0, 0 + mov r1, -2 + jsle r1, -3, +1 + jsle r1, -1, +1 + exit + mov32 r0, 1 + jsle r1, -2, +1 + mov32 r0, 2 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x1); +} + +#[test] +fn test_vm_jsle_reg() { + let prog = assemble( + " + mov32 r0, 0 + mov r1, -1 + mov r2, -2 + mov32 r3, 0 + jsle r1, r2, +1 + jsle r1, r3, +1 + exit + mov32 r0, 1 + mov r1, r2 + jsle r1, r2, +1 + mov32 r0, 2 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x1); +} + +#[test] +fn test_vm_jsgt_imm() { + let prog = assemble( + " + mov32 r0, 0 + mov r1, -2 + jsgt r1, -1, +4 + mov32 r0, 1 + mov32 r1, 0 + jsgt r1, -1, +1 + mov32 r0, 2 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x1); +} + +#[test] +fn test_vm_jsgt_reg() { + let prog = assemble( + " + mov32 r0, 0 + mov r1, -2 + mov r2, -1 + jsgt r1, r2, +4 + mov32 r0, 1 + mov32 r1, 0 + jsgt r1, r2, +1 + mov32 r0, 2 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x1); +} + +#[test] +fn test_vm_jslt_imm() { + let prog = assemble( + " + mov32 r0, 0 + mov r1, -2 + jslt r1, -3, +2 + jslt r1, -2, +1 + jslt r1, -1, +1 + exit + mov32 r0, 1 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x1); +} + +#[test] +fn test_vm_jslt_reg() { + let prog = assemble( + " + mov32 r0, 0 + mov r1, -2 + mov r2, -3 + mov r3, -1 + jslt r1, r1, +2 + jslt r1, r2, +1 + jslt r1, r3, +1 + exit + mov32 r0, 1 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x1); +} + +#[test] +fn test_vm_jeq32_imm() { + let prog = assemble( + " + mov r9, 1 + lsh r9, 32 + mov32 r0, 0x0 + mov32 r1, 0xa + jeq32 r1, 0xb, +5 + mov32 r0, 1 + mov r1, 0xb + or r1, r9 + jeq32 r1, 0xb, +1 + mov32 r0, 2 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x1); +} + +#[test] +fn test_vm_jeq32_reg() { + let prog = assemble( + " + mov r9, 1 + lsh r9, 32 + mov32 r0, 0 + mov32 r1, 0xa + mov32 r2, 0xb + jeq32 r1, r2, +5 + mov32 r0, 1 + mov32 r1, 0xb + or r1, r9 + jeq32 r1, r2, +1 + mov32 r0, 2 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x1); +} + +#[test] +fn test_vm_jge32_imm() { + let prog = assemble( + " + mov r9, 1 + lsh r9, 32 + mov32 r0, 0 + mov32 r1, 0xa + jge32 r1, 0xb, +5 + mov32 r0, 1 + or r1, r9 + mov32 r1, 0xc + jge32 r1, 0xb, +1 + mov32 r0, 2 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x1); +} + +#[test] +fn test_vm_jge32_reg() { + let prog = assemble( + " + mov r9, 1 + lsh r9, 32 + mov32 r0, 0 + mov32 r1, 0xa + mov32 r2, 0xb + jge32 r1, r2, +5 + mov32 r0, 1 + or r1, r9 + mov32 r1, 0xc + jge32 r1, r2, +1 + mov32 r0, 2 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x1); +} + +#[test] +fn test_vm_jgt32_imm() { + let prog = assemble( + " + mov r9, 1 + lsh r9, 32 + mov32 r0, 0 + mov32 r1, 5 + or r1, r9 + jgt32 r1, 6, +4 + jgt32 r1, 5, +3 + jgt32 r1, 4, +1 + exit + mov32 r0, 1 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x1); +} + +#[test] +fn test_vm_jgt32_reg() { + let prog = assemble( + " + mov r9, 1 + lsh r9, 32 + mov r0, 0 + mov r1, 5 + mov32 r1, 5 + or r1, r9 + mov r2, 6 + mov r3, 4 + jgt32 r1, r2, +4 + jgt32 r1, r1, +3 + jgt32 r1, r3, +1 + exit + mov r0, 1 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x1); +} + +#[test] +fn test_vm_jle32_imm() { + let prog = assemble( + " + mov r9, 1 + lsh r9, 32 + mov32 r0, 0 + mov32 r1, 5 + or r1, r9 + jle32 r1, 4, +5 + jle32 r1, 6, +1 + exit + jle32 r1, 5, +1 + exit + mov32 r0, 1 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x1); +} + +#[test] +fn test_vm_jle32_reg() { + let prog = assemble( + " + mov r9, 1 + lsh r9, 32 + mov r0, 0 + mov r1, 5 + mov r2, 4 + mov r3, 6 + or r1, r9 + jle32 r1, r2, +5 + jle32 r1, r1, +1 + exit + jle32 r1, r3, +1 + exit + mov r0, 1 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x1); +} + +#[test] +fn test_vm_jlt32_imm() { + let prog = assemble( + " + mov r9, 1 + lsh r9, 32 + mov32 r0, 0 + mov32 r1, 5 + or r1, r9 + jlt32 r1, 4, +4 + jlt32 r1, 5, +3 + jlt32 r1, 6, +1 + exit + mov32 r0, 1 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x1); +} + +#[test] +fn test_vm_jlt32_reg() { + let prog = assemble( + " + mov r9, 1 + lsh r9, 32 + mov r0, 0 + mov r1, 5 + mov r2, 4 + mov r3, 6 + or r1, r9 + jlt32 r1, r2, +4 + jlt32 r1, r1, +3 + jlt32 r1, r3, +1 + exit + mov r0, 1 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x1); +} + +#[test] +fn test_vm_jne32_imm() { + let prog = assemble( + " + mov r9, 1 + lsh r9, 32 + mov32 r0, 0 + mov32 r1, 0xb + or r1, r9 + jne32 r1, 0xb, +4 + mov32 r0, 1 + mov32 r1, 0xa + or r1, r9 + jne32 r1, 0xb, +1 + mov32 r0, 2 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x1); +} + +#[test] +fn test_vm_jne32_reg() { + let prog = assemble( + " + mov r9, 1 + lsh r9, 32 + mov32 r0, 0 + mov32 r1, 0xb + or r1, r9 + mov32 r2, 0xb + jne32 r1, r2, +4 + mov32 r0, 1 + mov32 r1, 0xa + or r1, r9 + jne32 r1, r2, +1 + mov32 r0, 2 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x1); +} + +#[test] +fn test_vm_jset32_imm() { + let prog = assemble( + " + mov r9, 1 + lsh r9, 32 + mov32 r0, 0 + mov32 r1, 0x7 + or r1, r9 + jset32 r1, 0x8, +4 + mov32 r0, 1 + mov32 r1, 0x9 + jset32 r1, 0x8, +1 + mov32 r0, 2 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x1); +} + +#[test] +fn test_vm_jset32_reg() { + let prog = assemble( + " + mov r9, 1 + lsh r9, 32 + mov32 r0, 0 + mov32 r1, 0x7 + or r1, r9 + mov32 r2, 0x8 + jset32 r1, r2, +4 + mov32 r0, 1 + mov32 r1, 0x9 + jset32 r1, r2, +1 + mov32 r0, 2 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x1); +} + +#[test] +fn test_vm_jsge32_imm() { + let prog = assemble( + " + mov r9, 1 + lsh r9, 32 + mov32 r0, 0 + mov32 r1, -2 + or r1, r9 + jsge32 r1, -1, +5 + jsge32 r1, 0, +4 + mov32 r0, 1 + mov r1, -1 + jsge32 r1, -1, +1 + mov32 r0, 2 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x1); +} + +#[test] +fn test_vm_jsge32_reg() { + let prog = assemble( + " + mov r9, 1 + lsh r9, 32 + mov32 r0, 0 + mov32 r1, -2 + or r1, r9 + mov r2, -1 + mov32 r3, 0 + jsge32 r1, r2, +5 + jsge32 r1, r3, +4 + mov32 r0, 1 + mov r1, r2 + jsge32 r1, r2, +1 + mov32 r0, 2 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x1); +} + +#[test] +fn test_vm_jsgt32_imm() { + let prog = assemble( + " + mov r9, 1 + lsh r9, 32 + mov32 r0, 0 + mov32 r1, -2 + or r1, r9 + jsgt32 r1, -1, +4 + mov32 r0, 1 + mov32 r1, 0 + jsgt32 r1, -1, +1 + mov32 r0, 2 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x1); +} + +#[test] +fn test_vm_jsgt32_reg() { + let prog = assemble( + " + mov r9, 1 + lsh r9, 32 + mov32 r0, 0 + mov32 r1, -2 + or r1, r9 + mov r2, -1 + jsgt32 r1, r2, +4 + mov32 r0, 1 + mov32 r1, 0 + jsgt32 r1, r2, +1 + mov32 r0, 2 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x1); +} + +#[test] +fn test_vm_jsle32_imm() { + let prog = assemble( + " + mov r9, 1 + lsh r9, 32 + mov32 r0, 0 + mov32 r1, -2 + or r1, r9 + jsle32 r1, -3, +5 + jsle32 r1, -1, +1 + exit + mov32 r0, 1 + jsle32 r1, -2, +1 + mov32 r0, 2 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x1); +} + +#[test] +fn test_vm_jsle32_reg() { + let prog = assemble( + " + mov r9, 1 + lsh r9, 32 + mov32 r0, 0 + mov32 r1, -2 + or r1, r9 + mov r2, -3 + mov32 r3, 0 + jsle32 r1, r2, +6 + jsle32 r1, r3, +1 + exit + mov32 r0, 1 + mov r1, r2 + jsle32 r1, r2, +1 + mov32 r0, 2 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x1); +} + +#[test] +fn test_vm_jslt32_imm() { + let prog = assemble( + " + mov r9, 1 + lsh r9, 32 + mov32 r0, 0 + mov32 r1, -2 + or r1, r9 + jslt32 r1, -3, +4 + jslt32 r1, -2, +3 + jslt32 r1, -1, +1 + exit + mov32 r0, 1 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x1); +} + +#[test] +fn test_vm_jslt32_reg() { + let prog = assemble( + " + mov r9, 1 + lsh r9, 32 + mov32 r0, 0 + mov32 r1, -2 + or r1, r9 + mov r2, -3 + mov r3, -1 + jslt32 r1, r1, +4 + jslt32 r1, r2, +3 + jslt32 r1, r3, +1 + exit + mov32 r0, 1 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x1); +} + +#[test] +fn test_vm_lddw() { + let prog = assemble( + "lddw r0, 0x1122334455667788 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x1122334455667788); +} + +#[test] +fn test_vm_lddw2() { + let prog = assemble( + " + lddw r0, 0x0000000080000000 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x80000000); +} + +#[test] +fn test_vm_ldxb_all() { + let prog = assemble( + " + mov r0, r1 + ldxb r9, [r0+0] + lsh r9, 0 + ldxb r8, [r0+1] + lsh r8, 4 + ldxb r7, [r0+2] + lsh r7, 8 + ldxb r6, [r0+3] + lsh r6, 12 + ldxb r5, [r0+4] + lsh r5, 16 + ldxb r4, [r0+5] + lsh r4, 20 + ldxb r3, [r0+6] + lsh r3, 24 + ldxb r2, [r0+7] + lsh r2, 28 + ldxb r1, [r0+8] + lsh r1, 32 + ldxb r0, [r0+9] + lsh r0, 36 + or r0, r1 + or r0, r2 + or r0, r3 + or r0, r4 + or r0, r5 + or r0, r6 + or r0, r7 + or r0, r8 + or r0, r9 + exit", + ) + .unwrap(); + let mem = &mut [0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09]; + let vm = rbpf::EbpfVmRaw::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program(mem).unwrap(), 0x9876543210); +} + +#[test] +fn test_vm_ldxb() { + let prog = assemble( + " + ldxb r0, [r1+2] + exit", + ) + .unwrap(); + let mem = &mut [0xaa, 0xbb, 0x11, 0xcc, 0xdd]; + let vm = rbpf::EbpfVmRaw::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program(mem).unwrap(), 0x11); +} + +#[test] +fn test_vm_ldxdw() { + let prog = assemble( + " + ldxdw r0, [r1+2] + exit", + ) + .unwrap(); + let mem = &mut [ + 0xaa, 0xbb, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88, 0xcc, 0xdd, + ]; + let vm = rbpf::EbpfVmRaw::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program(mem).unwrap(), 0x8877665544332211); +} + +#[test] +fn test_vm_ldxh_all() { + let prog = assemble( + " + mov r0, r1 + ldxh r9, [r0+0] + be16 r9 + lsh r9, 0 + ldxh r8, [r0+2] + be16 r8 + lsh r8, 4 + ldxh r7, [r0+4] + be16 r7 + lsh r7, 8 + ldxh r6, [r0+6] + be16 r6 + lsh r6, 12 + ldxh r5, [r0+8] + be16 r5 + lsh r5, 16 + ldxh r4, [r0+10] + be16 r4 + lsh r4, 20 + ldxh r3, [r0+12] + be16 r3 + lsh r3, 24 + ldxh r2, [r0+14] + be16 r2 + lsh r2, 28 + ldxh r1, [r0+16] + be16 r1 + lsh r1, 32 + ldxh r0, [r0+18] + be16 r0 + lsh r0, 36 + or r0, r1 + or r0, r2 + or r0, r3 + or r0, r4 + or r0, r5 + or r0, r6 + or r0, r7 + or r0, r8 + or r0, r9 + exit", + ) + .unwrap(); + let mem = &mut [ + 0x00, 0x00, 0x00, 0x01, 0x00, 0x02, 0x00, 0x03, 0x00, 0x04, 0x00, 0x05, 0x00, 0x06, 0x00, + 0x07, 0x00, 0x08, 0x00, 0x09, + ]; + let vm = rbpf::EbpfVmRaw::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program(mem).unwrap(), 0x9876543210); +} + +#[test] +fn test_vm_ldxh_all2() { + let prog = assemble( + " + mov r0, r1 + ldxh r9, [r0+0] + be16 r9 + ldxh r8, [r0+2] + be16 r8 + ldxh r7, [r0+4] + be16 r7 + ldxh r6, [r0+6] + be16 r6 + ldxh r5, [r0+8] + be16 r5 + ldxh r4, [r0+10] + be16 r4 + ldxh r3, [r0+12] + be16 r3 + ldxh r2, [r0+14] + be16 r2 + ldxh r1, [r0+16] + be16 r1 + ldxh r0, [r0+18] + be16 r0 + or r0, r1 + or r0, r2 + or r0, r3 + or r0, r4 + or r0, r5 + or r0, r6 + or r0, r7 + or r0, r8 + or r0, r9 + exit", + ) + .unwrap(); + let mem = &mut [ + 0x00, 0x01, 0x00, 0x02, 0x00, 0x04, 0x00, 0x08, 0x00, 0x10, 0x00, 0x20, 0x00, 0x40, 0x00, + 0x80, 0x01, 0x00, 0x02, 0x00, + ]; + let vm = rbpf::EbpfVmRaw::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program(mem).unwrap(), 0x3ff); +} + +#[test] +fn test_vm_ldxh() { + let prog = assemble( + " + ldxh r0, [r1+2] + exit", + ) + .unwrap(); + let mem = &mut [0xaa, 0xbb, 0x11, 0x22, 0xcc, 0xdd]; + let vm = rbpf::EbpfVmRaw::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program(mem).unwrap(), 0x2211); +} + +#[test] +fn test_vm_ldxh_same_reg() { + let prog = assemble( + " + mov r0, r1 + sth [r0], 0x1234 + ldxh r0, [r0] + exit", + ) + .unwrap(); + let mem = &mut [0xff, 0xff]; + let vm = rbpf::EbpfVmRaw::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program(mem).unwrap(), 0x1234); +} + +#[test] +fn test_vm_ldxw_all() { + let prog = assemble( + " + mov r0, r1 + ldxw r9, [r0+0] + be32 r9 + ldxw r8, [r0+4] + be32 r8 + ldxw r7, [r0+8] + be32 r7 + ldxw r6, [r0+12] + be32 r6 + ldxw r5, [r0+16] + be32 r5 + ldxw r4, [r0+20] + be32 r4 + ldxw r3, [r0+24] + be32 r3 + ldxw r2, [r0+28] + be32 r2 + ldxw r1, [r0+32] + be32 r1 + ldxw r0, [r0+36] + be32 r0 + or r0, r1 + or r0, r2 + or r0, r3 + or r0, r4 + or r0, r5 + or r0, r6 + or r0, r7 + or r0, r8 + or r0, r9 + exit", + ) + .unwrap(); + let mem = &mut [ + 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x08, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x08, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, + ]; + let vm = rbpf::EbpfVmRaw::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program(mem).unwrap(), 0x030f0f); +} + +#[test] +fn test_vm_ldxw() { + let prog = assemble( + " + ldxw r0, [r1+2] + exit", + ) + .unwrap(); + let mem = &mut [0xaa, 0xbb, 0x11, 0x22, 0x33, 0x44, 0xcc, 0xdd]; + let vm = rbpf::EbpfVmRaw::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program(mem).unwrap(), 0x44332211); +} + +#[test] +fn test_vm_le16() { + let prog = assemble( + " + ldxh r0, [r1] + le16 r0 + exit", + ) + .unwrap(); + let mem = &mut [0x22, 0x11]; + let vm = rbpf::EbpfVmRaw::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program(mem).unwrap(), 0x1122); +} + +#[test] +fn test_vm_le32() { + let prog = assemble( + " + ldxw r0, [r1] + le32 r0 + exit", + ) + .unwrap(); + let mem = &mut [0x44, 0x33, 0x22, 0x11]; + let vm = rbpf::EbpfVmRaw::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program(mem).unwrap(), 0x11223344); +} + +#[test] +fn test_vm_le64() { + let prog = assemble( + " + ldxdw r0, [r1] + le64 r0 + exit", + ) + .unwrap(); + let mem = &mut [0x88, 0x77, 0x66, 0x55, 0x44, 0x33, 0x22, 0x11]; + let vm = rbpf::EbpfVmRaw::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program(mem).unwrap(), 0x1122334455667788); +} + +#[test] +fn test_vm_lsh_imm() { + let prog = assemble( + " + mov r0, 1 + lsh r0, 4 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x10); +} + +#[test] +fn test_vm_lsh_reg() { + let prog = assemble( + " + mov r0, 1 + mov r7, 4 + lsh r0, r7 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x10); +} + +#[test] +fn test_vm_lsh32_imm() { + let prog = assemble( + " + mov32 r0, 1 + lsh32 r0, 4 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x10); +} + +#[test] +fn test_vm_lsh32_reg() { + let prog = assemble( + " + mov32 r0, 1 + mov32 r7, 4 + lsh32 r0, r7 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x10); +} + +#[test] +fn test_vm_lsh_imm_overflow() { + let prog = assemble( + " + mov r0, 1 + lsh r0, 64 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x1); +} + +#[test] +fn test_vm_lsh_reg_overflow() { + let prog = assemble( + " + mov r0, 1 + mov r7, 64 + lsh r0, r7 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x1); +} + +#[test] +fn test_vm_lsh32_imm_overflow() { + let prog = assemble( + " + mov32 r0, 1 + lsh32 r0, 32 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x1); +} + +#[test] +fn test_vm_lsh32_reg_overflow() { + let prog = assemble( + " + mov32 r0, 1 + mov32 r7, 32 + lsh32 r0, r7 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x1); +} + +#[test] +fn test_vm_mod() { + let prog = assemble( + " + mov32 r0, 5748 + mod32 r0, 92 + mov32 r1, 13 + mod32 r0, r1 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x5); +} + +#[test] +fn test_vm_mod32() { + let prog = assemble( + " + lddw r0, 0x100000003 + mod32 r0, 3 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x0); +} + +#[test] +fn test_vm_mod64() { + let prog = assemble( + " + mov32 r0, -1316649930 + lsh r0, 32 + or r0, 0x100dc5c8 + mov32 r1, 0xdde263e + lsh r1, 32 + or r1, 0x3cbef7f3 + mod r0, r1 + mod r0, 0x658f1778 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x30ba5a04); +} + +#[test] +fn test_vm_mov() { + let prog = assemble( + " + mov32 r1, 1 + mov32 r0, r1 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x1); +} + +#[test] +fn test_vm_mul32_imm() { + let prog = assemble( + " + mov r0, 3 + mul32 r0, 4 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0xc); +} + +#[test] +fn test_vm_mul32_reg() { + let prog = assemble( + " + mov r0, 3 + mov r1, 4 + mul32 r0, r1 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0xc); +} + +#[test] +fn test_vm_mul32_reg_overflow() { + let prog = assemble( + " + mov r0, 0x40000001 + mov r1, 4 + mul32 r0, r1 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x4); +} + +#[test] +fn test_vm_mul64_imm() { + let prog = assemble( + " + mov r0, 0x40000001 + mul r0, 4 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x100000004); +} + +#[test] +fn test_vm_mul64_reg() { + let prog = assemble( + " + mov r0, 0x40000001 + mov r1, 4 + mul r0, r1 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x100000004); +} + +#[test] +fn test_vm_mul_loop() { + let prog = assemble( + " + mov r0, 0x7 + add r1, 0xa + lsh r1, 0x20 + rsh r1, 0x20 + jeq r1, 0x0, +4 + mov r0, 0x7 + mul r0, 0x7 + add r1, -1 + jne r1, 0x0, -3 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x75db9c97); +} + +#[test] +fn test_vm_neg64() { + let prog = assemble( + " + mov32 r0, 2 + neg r0 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0xfffffffffffffffe); +} + +#[test] +fn test_vm_neg() { + let prog = assemble( + " + mov32 r0, 2 + neg32 r0 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0xfffffffe); +} + +#[test] +fn test_vm_prime() { + let prog = assemble( + " + mov r1, 67 + mov r0, 0x1 + mov r2, 0x2 + jgt r1, 0x2, +4 + ja +10 + add r2, 0x1 + mov r0, 0x1 + jge r2, r1, +7 + mov r3, r1 + div r3, r2 + mul r3, r2 + mov r4, r1 + sub r4, r3 + mov r0, 0x0 + jne r4, 0x0, -10 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x1); +} + +#[test] +fn test_vm_rhs32() { + let prog = assemble( + " + xor r0, r0 + sub r0, 1 + rsh32 r0, 8 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x00ffffff); +} + +#[test] +fn test_vm_rsh_reg() { + let prog = assemble( + " + mov r0, 0x10 + mov r7, 4 + rsh r0, r7 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x1); +} + +#[test] +fn test_vm_stack() { + let prog = assemble( + " + mov r1, 51 + stdw [r10-16], 0xab + stdw [r10-8], 0xcd + and r1, 1 + lsh r1, 3 + mov r2, r10 + add r2, r1 + ldxdw r0, [r2-16] + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0xcd); +} + +#[test] +fn test_vm_stack2() { + let prog = assemble( + " + stb [r10-4], 0x01 + stb [r10-3], 0x02 + stb [r10-2], 0x03 + stb [r10-1], 0x04 + mov r1, r10 + mov r2, 0x4 + sub r1, r2 + call 1 + mov r1, 0 + ldxb r2, [r10-4] + ldxb r3, [r10-3] + ldxb r4, [r10-2] + ldxb r5, [r10-1] + call 0 + xor r0, 0x2a2a2a2a + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.register_helper(0, helpers::gather_bytes).unwrap(); + vm.register_helper(1, helpers::memfrob).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x01020304); +} + +#[test] +fn test_vm_stb() { + let prog = assemble( + " + stb [r1+2], 0x11 + ldxb r0, [r1+2] + exit", + ) + .unwrap(); + let mem = &mut [0xaa, 0xbb, 0xff, 0xcc, 0xdd]; + let vm = rbpf::EbpfVmRaw::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program(mem).unwrap(), 0x11); +} + +#[test] +fn test_vm_stdw() { + let prog = assemble( + " + stdw [r1+2], 0x44332211 + ldxdw r0, [r1+2] + exit", + ) + .unwrap(); + let mem = &mut [ + 0xaa, 0xbb, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xcc, 0xdd, + ]; + let vm = rbpf::EbpfVmRaw::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program(mem).unwrap(), 0x44332211); +} + +// If this case is not handled properly in check_mem(), then we may overflow when adding the +// context address and the offset, and make the thread panic with "attempt to add with overflow". +// Check that we panic with the expected out-of-bounds error. +#[test] +#[should_panic(expected = "Error: out of bounds memory store (insn #1)")] +fn test_vm_stdw_add_overflow() { + let prog = assemble( + " + stdw [r2-0x1], 0x44332211 + ldxw r0, [r1+2] + exit", + ) + .unwrap(); + let mem = &mut [ + 0xaa, 0xbb, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xcc, 0xdd, + ]; + let mut vm = rbpf::EbpfVmFixedMbuff::new(Some(&prog), 0x00, 0x10).unwrap(); + _ = vm.execute_program(mem).unwrap(); +} + +#[test] +fn test_vm_sth() { + let prog = assemble( + " + sth [r1+2], 0x2211 + ldxh r0, [r1+2] + exit", + ) + .unwrap(); + let mem = &mut [0xaa, 0xbb, 0xff, 0xff, 0xcc, 0xdd]; + let vm = rbpf::EbpfVmRaw::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program(mem).unwrap(), 0x2211); +} + +#[test] +fn test_vm_string_stack() { + let prog = assemble( + " + mov r1, 0x78636261 + stxw [r10-8], r1 + mov r6, 0x0 + stxb [r10-4], r6 + stxb [r10-12], r6 + mov r1, 0x79636261 + stxw [r10-16], r1 + mov r1, r10 + add r1, -8 + mov r2, r1 + call 0x4 + mov r1, r0 + mov r0, 0x1 + lsh r1, 0x20 + rsh r1, 0x20 + jne r1, 0x0, +11 + mov r1, r10 + add r1, -8 + mov r2, r10 + add r2, -16 + call 0x4 + mov r1, r0 + lsh r1, 0x20 + rsh r1, 0x20 + mov r0, 0x1 + jeq r1, r6, +1 + mov r0, 0x0 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.register_helper(4, helpers::strcmp).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x0); +} + +#[test] +fn test_vm_stw() { + let prog = assemble( + " + stw [r1+2], 0x44332211 + ldxw r0, [r1+2] + exit", + ) + .unwrap(); + let mem = &mut [0xaa, 0xbb, 0xff, 0xff, 0xff, 0xff, 0xcc, 0xdd]; + let vm = rbpf::EbpfVmRaw::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program(mem).unwrap(), 0x44332211); +} + +#[test] +fn test_vm_stxb() { + let prog = assemble( + " + mov32 r2, 0x11 + stxb [r1+2], r2 + ldxb r0, [r1+2] + exit", + ) + .unwrap(); + let mem = &mut [0xaa, 0xbb, 0xff, 0xcc, 0xdd]; + let vm = rbpf::EbpfVmRaw::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program(mem).unwrap(), 0x11); +} + +#[test] +fn test_vm_stxb_all() { + let prog = assemble( + " + mov r0, 0xf0 + mov r2, 0xf2 + mov r3, 0xf3 + mov r4, 0xf4 + mov r5, 0xf5 + mov r6, 0xf6 + mov r7, 0xf7 + mov r8, 0xf8 + stxb [r1], r0 + stxb [r1+1], r2 + stxb [r1+2], r3 + stxb [r1+3], r4 + stxb [r1+4], r5 + stxb [r1+5], r6 + stxb [r1+6], r7 + stxb [r1+7], r8 + ldxdw r0, [r1] + be64 r0 + exit", + ) + .unwrap(); + let mem = &mut [0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff]; + let vm = rbpf::EbpfVmRaw::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program(mem).unwrap(), 0xf0f2f3f4f5f6f7f8); +} + +#[test] +fn test_vm_stxb_all2() { + let prog = assemble( + " + mov r0, r1 + mov r1, 0xf1 + mov r9, 0xf9 + stxb [r0], r1 + stxb [r0+1], r9 + ldxh r0, [r0] + be16 r0 + exit", + ) + .unwrap(); + let mem = &mut [0xff, 0xff]; + let vm = rbpf::EbpfVmRaw::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program(mem).unwrap(), 0xf1f9); +} + +#[test] +fn test_vm_stxb_chain() { + let prog = assemble( + " + mov r0, r1 + ldxb r9, [r0+0] + stxb [r0+1], r9 + ldxb r8, [r0+1] + stxb [r0+2], r8 + ldxb r7, [r0+2] + stxb [r0+3], r7 + ldxb r6, [r0+3] + stxb [r0+4], r6 + ldxb r5, [r0+4] + stxb [r0+5], r5 + ldxb r4, [r0+5] + stxb [r0+6], r4 + ldxb r3, [r0+6] + stxb [r0+7], r3 + ldxb r2, [r0+7] + stxb [r0+8], r2 + ldxb r1, [r0+8] + stxb [r0+9], r1 + ldxb r0, [r0+9] + exit", + ) + .unwrap(); + let mem = &mut [0x2a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00]; + let vm = rbpf::EbpfVmRaw::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program(mem).unwrap(), 0x2a); +} + +#[test] +fn test_vm_stxdw() { + let prog = assemble( + " + mov r2, -2005440939 + lsh r2, 32 + or r2, 0x44332211 + stxdw [r1+2], r2 + ldxdw r0, [r1+2] + exit", + ) + .unwrap(); + let mem = &mut [ + 0xaa, 0xbb, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xcc, 0xdd, + ]; + let vm = rbpf::EbpfVmRaw::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program(mem).unwrap(), 0x8877665544332211); +} + +#[test] +fn test_vm_stxh() { + let prog = assemble( + " + mov32 r2, 0x2211 + stxh [r1+2], r2 + ldxh r0, [r1+2] + exit", + ) + .unwrap(); + let mem = &mut [0xaa, 0xbb, 0xff, 0xff, 0xcc, 0xdd]; + let vm = rbpf::EbpfVmRaw::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program(mem).unwrap(), 0x2211); +} + +#[test] +fn test_vm_stxw() { + let prog = assemble( + " + mov32 r2, 0x44332211 + stxw [r1+2], r2 + ldxw r0, [r1+2] + exit", + ) + .unwrap(); + let mem = &mut [0xaa, 0xbb, 0xff, 0xff, 0xff, 0xff, 0xcc, 0xdd]; + let vm = rbpf::EbpfVmRaw::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program(mem).unwrap(), 0x44332211); +} + +#[test] +fn test_vm_subnet() { + let prog = assemble( + " + mov r2, 0xe + ldxh r3, [r1+12] + jne r3, 0x81, +2 + mov r2, 0x12 + ldxh r3, [r1+16] + and r3, 0xffff + jne r3, 0x8, +5 + add r1, r2 + mov r0, 0x1 + ldxw r1, [r1+16] + and r1, 0xffffff + jeq r1, 0x1a8c0, +1 + mov r0, 0x0 + exit", + ) + .unwrap(); + let mem = &mut [ + 0x00, 0x00, 0xc0, 0x9f, 0xa0, 0x97, 0x00, 0xa0, 0xcc, 0x3b, 0xbf, 0xfa, 0x08, 0x00, 0x45, + 0x10, 0x00, 0x3c, 0x46, 0x3c, 0x40, 0x00, 0x40, 0x06, 0x73, 0x1c, 0xc0, 0xa8, 0x01, 0x02, + 0xc0, 0xa8, 0x01, 0x01, 0x06, 0x0e, 0x00, 0x17, 0x99, 0xc5, 0xa0, 0xec, 0x00, 0x00, 0x00, + 0x00, 0xa0, 0x02, 0x7d, 0x78, 0xe0, 0xa3, 0x00, 0x00, 0x02, 0x04, 0x05, 0xb4, 0x04, 0x02, + 0x08, 0x0a, 0x00, 0x9c, 0x27, 0x24, 0x00, 0x00, 0x00, 0x00, 0x01, 0x03, 0x03, 0x00, + ]; + let vm = rbpf::EbpfVmRaw::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program(mem).unwrap(), 0x1); +} + +const PROG_TCP_PORT_80: [u8; 152] = [ + 0x71, 0x12, 0x0c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x71, 0x13, 0x0d, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x67, 0x03, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x4f, 0x23, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xb7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x55, 0x03, 0x0c, 0x00, 0x08, 0x00, 0x00, 0x00, + 0x71, 0x12, 0x17, 0x00, 0x00, 0x00, 0x00, 0x00, 0x55, 0x02, 0x0a, 0x00, 0x06, 0x00, 0x00, 0x00, + 0x71, 0x12, 0x0e, 0x00, 0x00, 0x00, 0x00, 0x00, 0x07, 0x01, 0x00, 0x00, 0x0e, 0x00, 0x00, 0x00, + 0x57, 0x02, 0x00, 0x00, 0x0f, 0x00, 0x00, 0x00, 0x67, 0x02, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, + 0x0f, 0x21, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x69, 0x12, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x15, 0x02, 0x02, 0x00, 0x00, 0x50, 0x00, 0x00, 0x69, 0x11, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x55, 0x01, 0x01, 0x00, 0x00, 0x50, 0x00, 0x00, 0xb7, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +]; + +#[test] +fn test_vm_tcp_port80_match() { + let mem = &mut [ + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x00, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x08, 0x00, 0x45, + 0x00, 0x00, 0x56, 0x00, 0x01, 0x00, 0x00, 0x40, 0x06, 0xf9, 0x4d, 0xc0, 0xa8, 0x00, 0x01, + 0xc0, 0xa8, 0x00, 0x02, 0x27, 0x10, 0x00, 0x50, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x50, 0x02, 0x20, 0x00, 0xc5, 0x18, 0x00, 0x00, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, + 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, + 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, + 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, + ]; + let prog = &PROG_TCP_PORT_80; + let vm = rbpf::EbpfVmRaw::new(Some(prog)).unwrap(); + assert_eq!(vm.execute_program(mem).unwrap(), 0x1); +} + +#[test] +fn test_vm_tcp_port80_nomatch() { + let mem = &mut [ + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x00, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x08, 0x00, 0x45, + 0x00, 0x00, 0x56, 0x00, 0x01, 0x00, 0x00, 0x40, 0x06, 0xf9, 0x4d, 0xc0, 0xa8, 0x00, 0x01, + 0xc0, 0xa8, 0x00, 0x02, 0x00, 0x16, 0x27, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x51, 0x02, 0x20, 0x00, 0xc5, 0x18, 0x00, 0x00, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, + 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, + 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, + 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, + ]; + let prog = &PROG_TCP_PORT_80; + let vm = rbpf::EbpfVmRaw::new(Some(prog)).unwrap(); + assert_eq!(vm.execute_program(mem).unwrap(), 0x0); +} + +#[test] +fn test_vm_tcp_port80_nomatch_ethertype() { + let mem = &mut [ + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x00, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x08, 0x01, 0x45, + 0x00, 0x00, 0x56, 0x00, 0x01, 0x00, 0x00, 0x40, 0x06, 0xf9, 0x4d, 0xc0, 0xa8, 0x00, 0x01, + 0xc0, 0xa8, 0x00, 0x02, 0x27, 0x10, 0x00, 0x50, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x50, 0x02, 0x20, 0x00, 0xc5, 0x18, 0x00, 0x00, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, + 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, + 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, + 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, + ]; + let prog = &PROG_TCP_PORT_80; + let vm = rbpf::EbpfVmRaw::new(Some(prog)).unwrap(); + assert_eq!(vm.execute_program(mem).unwrap(), 0x0); +} + +#[test] +fn test_vm_tcp_port80_nomatch_proto() { + let mem = &mut [ + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x00, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x08, 0x00, 0x45, + 0x00, 0x00, 0x56, 0x00, 0x01, 0x00, 0x00, 0x40, 0x11, 0xf9, 0x4d, 0xc0, 0xa8, 0x00, 0x01, + 0xc0, 0xa8, 0x00, 0x02, 0x27, 0x10, 0x00, 0x50, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x50, 0x02, 0x20, 0x00, 0xc5, 0x18, 0x00, 0x00, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, + 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, + 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, + 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, + ]; + let prog = &PROG_TCP_PORT_80; + let vm = rbpf::EbpfVmRaw::new(Some(prog)).unwrap(); + assert_eq!(vm.execute_program(mem).unwrap(), 0x0); +} + +#[test] +fn test_vm_tcp_sack_match() { + let mut mem = TCP_SACK_MATCH.to_vec(); + let prog = assemble(TCP_SACK_ASM).unwrap(); + let vm = rbpf::EbpfVmRaw::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program(mem.as_mut_slice()).unwrap(), 0x1); +} + +#[test] +fn test_vm_tcp_sack_nomatch() { + let mut mem = TCP_SACK_NOMATCH.to_vec(); + let prog = assemble(TCP_SACK_ASM).unwrap(); + let vm = rbpf::EbpfVmRaw::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program(mem.as_mut_slice()).unwrap(), 0x0); +} diff --git a/kernel/src/arch/riscv64/interrupt/handle.rs b/kernel/src/arch/riscv64/interrupt/handle.rs index faff690f2..fc9716798 100644 --- a/kernel/src/arch/riscv64/interrupt/handle.rs +++ b/kernel/src/arch/riscv64/interrupt/handle.rs @@ -3,12 +3,12 @@ //! 架构相关的处理逻辑参考： https://code.dragonos.org.cn/xref/linux-6.6.21/arch/riscv/kernel/traps.c use core::hint::spin_loop; -use log::error; +use log::{error, trace}; use system_error::SystemError; -use crate::{arch::syscall::syscall_handler, driver::irqchip::riscv_intc::riscv_intc_irq}; - use super::TrapFrame; +use crate::exception::ebreak::EBreak; +use crate::{arch::syscall::syscall_handler, driver::irqchip::riscv_intc::riscv_intc_irq}; type ExceptionHandler = fn(&mut TrapFrame) -> Result<(), SystemError>; @@ -93,11 +93,10 @@ fn do_trap_insn_illegal(_trap_frame: &mut TrapFrame) -> Result<(), SystemError> } /// 处理断点异常 #3 -fn do_trap_break(_trap_frame: &mut TrapFrame) -> Result<(), SystemError> { - error!("riscv64_do_irq: do_trap_break"); - loop { - spin_loop(); - } +fn do_trap_break(trap_frame: &mut TrapFrame) -> Result<(), SystemError> { + trace!("riscv64_do_irq: do_trap_break"); + // handle breakpoint + EBreak::handle(trap_frame) } /// 处理加载地址不对齐异常 #4 diff --git a/kernel/src/arch/riscv64/interrupt/mod.rs b/kernel/src/arch/riscv64/interrupt/mod.rs index dc6d7893a..bc366098f 100644 --- a/kernel/src/arch/riscv64/interrupt/mod.rs +++ b/kernel/src/arch/riscv64/interrupt/mod.rs @@ -1,3 +1,5 @@ +use core::any::Any; +use kprobe::ProbeArgs; use riscv::register::{scause::Scause, sstatus::Sstatus}; use system_error::SystemError; @@ -160,4 +162,21 @@ impl TrapFrame { pub fn set_return_value(&mut self, value: usize) { self.a0 = value; } + + /// 设置当前的程序计数器 + pub fn set_pc(&mut self, pc: usize) { + self.epc = pc; + } +} + +impl ProbeArgs for TrapFrame { + fn as_any(&self) -> &dyn Any { + self + } + fn break_address(&self) -> usize { + self.epc + } + fn debug_address(&self) -> usize { + self.epc + } } diff --git a/kernel/src/arch/riscv64/kprobe.rs b/kernel/src/arch/riscv64/kprobe.rs new file mode 100644 index 000000000..960b06cd6 --- /dev/null +++ b/kernel/src/arch/riscv64/kprobe.rs @@ -0,0 +1,85 @@ +use crate::arch::interrupt::TrapFrame; + +pub fn setup_single_step(frame: &mut TrapFrame, step_addr: usize) { + frame.set_pc(step_addr); +} + +pub fn clear_single_step(frame: &mut TrapFrame, return_addr: usize) { + frame.set_pc(return_addr); +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct KProbeContext { + pub pc: usize, + pub ra: usize, + pub sp: usize, + pub gp: usize, + pub tp: usize, + pub t0: usize, + pub t1: usize, + pub t2: usize, + pub s0: usize, + pub s1: usize, + pub a0: usize, + pub a1: usize, + pub a2: usize, + pub a3: usize, + pub a4: usize, + pub a5: usize, + pub a6: usize, + pub a7: usize, + pub s2: usize, + pub s3: usize, + pub s4: usize, + pub s5: usize, + pub s6: usize, + pub s7: usize, + pub s8: usize, + pub s9: usize, + pub s10: usize, + pub s11: usize, + pub t3: usize, + pub t4: usize, + pub t5: usize, + pub t6: usize, +} + +impl From<&TrapFrame> for KProbeContext { + fn from(trap_frame: &TrapFrame) -> Self { + Self { + pc: trap_frame.epc, + ra: trap_frame.ra, + sp: trap_frame.sp, + gp: trap_frame.gp, + tp: trap_frame.tp, + t0: trap_frame.t0, + t1: trap_frame.t1, + t2: trap_frame.t2, + s0: trap_frame.s0, + s1: trap_frame.s1, + a0: trap_frame.a0, + a1: trap_frame.a1, + a2: trap_frame.a2, + a3: trap_frame.a3, + a4: trap_frame.a4, + a5: trap_frame.a5, + a6: trap_frame.a6, + a7: trap_frame.a7, + s2: trap_frame.s2, + s3: trap_frame.s3, + s4: trap_frame.s4, + s5: trap_frame.s5, + s6: trap_frame.s6, + s7: trap_frame.s7, + s8: trap_frame.s8, + s9: trap_frame.s9, + s10: trap_frame.s10, + s11: trap_frame.s11, + t3: trap_frame.t3, + t4: trap_frame.t4, + t5: trap_frame.t5, + t6: trap_frame.t6, + } + } +} diff --git a/kernel/src/arch/riscv64/mod.rs b/kernel/src/arch/riscv64/mod.rs index e7c2fe222..a34b99a7f 100644 --- a/kernel/src/arch/riscv64/mod.rs +++ b/kernel/src/arch/riscv64/mod.rs @@ -5,6 +5,7 @@ pub mod elf; pub mod init; pub mod interrupt; pub mod ipc; +pub mod kprobe; mod kvm; pub mod mm; pub mod msi; diff --git a/kernel/src/arch/x86_64/interrupt/mod.rs b/kernel/src/arch/x86_64/interrupt/mod.rs index 0eb4a88a8..e83566ca7 100644 --- a/kernel/src/arch/x86_64/interrupt/mod.rs +++ b/kernel/src/arch/x86_64/interrupt/mod.rs @@ -4,11 +4,12 @@ pub mod ipi; pub mod msi; pub mod trap; +use core::any::Any; use core::{ arch::asm, sync::atomic::{compiler_fence, Ordering}, }; - +use kprobe::ProbeArgs; use log::error; use system_error::SystemError; @@ -177,4 +178,21 @@ impl TrapFrame { pub fn is_from_user(&self) -> bool { return (self.cs & 0x3) != 0; } + /// 设置当前的程序计数器 + pub fn set_pc(&mut self, pc: usize) { + self.rip = pc as u64; + } +} + +impl ProbeArgs for TrapFrame { + fn as_any(&self) -> &dyn Any { + self + } + fn break_address(&self) -> usize { + (self.rip - 1) as usize + } + + fn debug_address(&self) -> usize { + self.rip as usize + } } diff --git a/kernel/src/arch/x86_64/interrupt/trap.rs b/kernel/src/arch/x86_64/interrupt/trap.rs index 9cd31fa9b..6cb3971a5 100644 --- a/kernel/src/arch/x86_64/interrupt/trap.rs +++ b/kernel/src/arch/x86_64/interrupt/trap.rs @@ -1,6 +1,12 @@ -use log::{error, warn}; +use log::{error, trace, warn}; use system_error::SystemError; +use super::{ + entry::{set_intr_gate, set_system_trap_gate}, + TrapFrame, +}; +use crate::exception::debug::DebugException; +use crate::exception::ebreak::EBreak; use crate::{ arch::{CurrentIrqArch, MMArch}, exception::InterruptArch, @@ -9,11 +15,6 @@ use crate::{ smp::core::smp_get_processor_id, }; -use super::{ - entry::{set_intr_gate, set_system_trap_gate}, - TrapFrame, -}; - extern "C" { fn trap_divide_error(); fn trap_debug(); @@ -125,8 +126,8 @@ unsafe extern "C" fn do_divide_error(regs: &'static TrapFrame, error_code: u64) /// 处理调试异常 1 #DB #[no_mangle] -unsafe extern "C" fn do_debug(regs: &'static TrapFrame, error_code: u64) { - error!( +unsafe extern "C" fn do_debug(regs: &'static mut TrapFrame, error_code: u64) { + trace!( "do_debug(1), \tError code: {:#x},\trsp: {:#x},\trip: {:#x},\t CPU: {}, \tpid: {:?}", error_code, regs.rsp, @@ -134,7 +135,7 @@ unsafe extern "C" fn do_debug(regs: &'static TrapFrame, error_code: u64) { smp_get_processor_id().data(), ProcessManager::current_pid() ); - panic!("Debug Exception"); + DebugException::handle(regs).unwrap(); } /// 处理NMI中断 2 NMI @@ -153,8 +154,8 @@ unsafe extern "C" fn do_nmi(regs: &'static TrapFrame, error_code: u64) { /// 处理断点异常 3 #BP #[no_mangle] -unsafe extern "C" fn do_int3(regs: &'static TrapFrame, error_code: u64) { - error!( +unsafe extern "C" fn do_int3(regs: &'static mut TrapFrame, error_code: u64) { + trace!( "do_int3(3), \tError code: {:#x},\trsp: {:#x},\trip: {:#x},\t CPU: {}, \tpid: {:?}", error_code, regs.rsp, @@ -162,7 +163,7 @@ unsafe extern "C" fn do_int3(regs: &'static TrapFrame, error_code: u64) { smp_get_processor_id().data(), ProcessManager::current_pid() ); - panic!("Int3"); + EBreak::handle(regs).unwrap(); } /// 处理溢出异常 4 #OF diff --git a/kernel/src/arch/x86_64/kprobe.rs b/kernel/src/arch/x86_64/kprobe.rs new file mode 100644 index 000000000..e998aa993 --- /dev/null +++ b/kernel/src/arch/x86_64/kprobe.rs @@ -0,0 +1,65 @@ +use crate::arch::interrupt::TrapFrame; + +pub fn setup_single_step(frame: &mut TrapFrame, step_addr: usize) { + frame.rflags |= 0x100; + frame.set_pc(step_addr); +} + +pub fn clear_single_step(frame: &mut TrapFrame, return_addr: usize) { + frame.rflags &= !0x100; + frame.set_pc(return_addr); +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct KProbeContext { + pub r15: ::core::ffi::c_ulong, + pub r14: ::core::ffi::c_ulong, + pub r13: ::core::ffi::c_ulong, + pub r12: ::core::ffi::c_ulong, + pub rbp: ::core::ffi::c_ulong, + pub rbx: ::core::ffi::c_ulong, + pub r11: ::core::ffi::c_ulong, + pub r10: ::core::ffi::c_ulong, + pub r9: ::core::ffi::c_ulong, + pub r8: ::core::ffi::c_ulong, + pub rax: ::core::ffi::c_ulong, + pub rcx: ::core::ffi::c_ulong, + pub rdx: ::core::ffi::c_ulong, + pub rsi: ::core::ffi::c_ulong, + pub rdi: ::core::ffi::c_ulong, + pub orig_rax: ::core::ffi::c_ulong, + pub rip: ::core::ffi::c_ulong, + pub cs: ::core::ffi::c_ulong, + pub eflags: ::core::ffi::c_ulong, + pub rsp: ::core::ffi::c_ulong, + pub ss: ::core::ffi::c_ulong, +} + +impl From<&TrapFrame> for KProbeContext { + fn from(trap_frame: &TrapFrame) -> Self { + Self { + r15: trap_frame.r15, + r14: trap_frame.r14, + r13: trap_frame.r13, + r12: trap_frame.r12, + rbp: trap_frame.rbp, + rbx: trap_frame.rbx, + r11: trap_frame.r11, + r10: trap_frame.r10, + r9: trap_frame.r9, + r8: trap_frame.r8, + rax: trap_frame.rax, + rcx: trap_frame.rcx, + rdx: trap_frame.rdx, + rsi: trap_frame.rsi, + rdi: trap_frame.rdi, + orig_rax: 0, + rip: trap_frame.rip, + cs: trap_frame.cs, + eflags: trap_frame.rflags, + rsp: trap_frame.rsp, + ss: trap_frame.ss, + } + } +} diff --git a/kernel/src/arch/x86_64/mod.rs b/kernel/src/arch/x86_64/mod.rs index ff95ab1d1..2bd97fe00 100644 --- a/kernel/src/arch/x86_64/mod.rs +++ b/kernel/src/arch/x86_64/mod.rs @@ -8,6 +8,7 @@ pub mod fpu; pub mod init; pub mod interrupt; pub mod ipc; +pub mod kprobe; pub mod kvm; pub mod libs; pub mod mm; diff --git a/kernel/src/bpf/helper/consts.rs b/kernel/src/bpf/helper/consts.rs new file mode 100644 index 000000000..69bce7d61 --- /dev/null +++ b/kernel/src/bpf/helper/consts.rs @@ -0,0 +1,11 @@ +pub const HELPER_MAP_LOOKUP_ELEM: u32 = 1; +pub const HELPER_MAP_UPDATE_ELEM: u32 = 2; +pub const HELPER_MAP_DELETE_ELEM: u32 = 3; +pub const HELPER_MAP_FOR_EACH_ELEM: u32 = 164; +pub const HELPER_MAP_LOOKUP_PERCPU_ELEM: u32 = 195; +pub const HELPER_PERF_EVENT_OUTPUT: u32 = 25; +pub const HELPER_BPF_PROBE_READ: u32 = 4; +pub const HELPER_TRACE_PRINTF: u32 = 6; +pub const HELPER_MAP_PUSH_ELEM: u32 = 87; +pub const HELPER_MAP_POP_ELEM: u32 = 88; +pub const HELPER_MAP_PEEK_ELEM: u32 = 89; diff --git a/kernel/src/bpf/helper/mod.rs b/kernel/src/bpf/helper/mod.rs new file mode 100644 index 000000000..2f951793f --- /dev/null +++ b/kernel/src/bpf/helper/mod.rs @@ -0,0 +1,340 @@ +mod consts; +mod print; + +use crate::bpf::helper::print::trace_printf; +use crate::bpf::map::{BpfCallBackFn, BpfMap}; +use crate::include::bindings::linux_bpf::BPF_F_CURRENT_CPU; +use crate::libs::lazy_init::Lazy; +use crate::smp::core::smp_get_processor_id; +use alloc::{collections::BTreeMap, sync::Arc}; +use core::ffi::c_void; +use system_error::SystemError; + +type RawBPFHelperFn = fn(u64, u64, u64, u64, u64) -> u64; +type Result = core::result::Result; +macro_rules! define_func { + ($name:ident) => { + core::mem::transmute::($name as usize) + }; +} + +/// See https://ebpf-docs.dylanreimerink.nl/linux/helper-function/bpf_map_lookup_elem/ +unsafe fn raw_map_lookup_elem(map: *mut c_void, key: *const c_void) -> *const c_void { + let map = Arc::from_raw(map as *const BpfMap); + let key_size = map.key_size(); + let key = core::slice::from_raw_parts(key as *const u8, key_size); + let value = map_lookup_elem(&map, key); + // log::info!(": {:x?}", value); + // warning: We need to keep the map alive, so we don't drop it here. + let _ = Arc::into_raw(map); + match value { + Ok(Some(value)) => value as *const c_void, + _ => core::ptr::null_mut(), + } +} + +pub fn map_lookup_elem(map: &Arc, key: &[u8]) -> Result> { + let mut binding = map.inner_map().lock(); + let value = binding.lookup_elem(key); + match value { + Ok(Some(value)) => Ok(Some(value.as_ptr())), + _ => Ok(None), + } +} + +/// See https://ebpf-docs.dylanreimerink.nl/linux/helper-function/bpf_perf_event_output/ +/// +/// See https://man7.org/linux/man-pages/man7/bpf-helpers.7.html +unsafe fn raw_perf_event_output( + ctx: *mut c_void, + map: *mut c_void, + flags: u64, + data: *mut c_void, + size: u64, +) -> i64 { + // log::info!(": {:x?}", data); + let map = Arc::from_raw(map as *const BpfMap); + let data = core::slice::from_raw_parts(data as *const u8, size as usize); + let res = perf_event_output(ctx, &map, flags, data); + // warning: We need to keep the map alive, so we don't drop it here. + let _ = Arc::into_raw(map); + match res { + Ok(_) => 0, + Err(e) => e as i64, + } +} + +pub fn perf_event_output( + ctx: *mut c_void, + map: &Arc, + flags: u64, + data: &[u8], +) -> Result<()> { + let mut binding = map.inner_map().lock(); + let index = flags as u32; + let flags = (flags >> 32) as u32; + let key = if index == BPF_F_CURRENT_CPU as u32 { + smp_get_processor_id().data() + } else { + index + }; + let fd = binding + .lookup_elem(&key.to_ne_bytes())? + .ok_or(SystemError::ENOENT)?; + let fd = u32::from_ne_bytes(fd.try_into().map_err(|_| SystemError::EINVAL)?); + crate::perf::perf_event_output(ctx, fd as usize, flags, data)?; + Ok(()) +} + +/// See https://ebpf-docs.dylanreimerink.nl/linux/helper-function/bpf_probe_read/ +fn raw_bpf_probe_read(dst: *mut c_void, size: u32, unsafe_ptr: *const c_void) -> i64 { + log::info!( + "raw_bpf_probe_read, dst:{:x}, size:{}, unsafe_ptr: {:x}", + dst as usize, + size, + unsafe_ptr as usize + ); + let (dst, src) = unsafe { + let dst = core::slice::from_raw_parts_mut(dst as *mut u8, size as usize); + let src = core::slice::from_raw_parts(unsafe_ptr as *const u8, size as usize); + (dst, src) + }; + let res = bpf_probe_read(dst, src); + match res { + Ok(_) => 0, + Err(e) => e as i64, + } +} + +/// For tracing programs, safely attempt to read size +/// bytes from kernel space address unsafe_ptr and +/// store the data in dst. +pub fn bpf_probe_read(dst: &mut [u8], src: &[u8]) -> Result<()> { + log::info!("bpf_probe_read: len: {}", dst.len()); + dst.copy_from_slice(src); + Ok(()) +} + +unsafe fn raw_map_update_elem( + map: *mut c_void, + key: *const c_void, + value: *const c_void, + flags: u64, +) -> i64 { + let map = Arc::from_raw(map as *const BpfMap); + let key_size = map.key_size(); + let value_size = map.value_size(); + // log::info!(": flags: {:x?}", flags); + let key = core::slice::from_raw_parts(key as *const u8, key_size); + let value = core::slice::from_raw_parts(value as *const u8, value_size); + let res = map_update_elem(&map, key, value, flags); + let _ = Arc::into_raw(map); + match res { + Ok(_) => 0, + Err(e) => e as _, + } +} + +pub fn map_update_elem(map: &Arc, key: &[u8], value: &[u8], flags: u64) -> Result<()> { + let mut binding = map.inner_map().lock(); + let value = binding.update_elem(key, value, flags); + value +} + +/// Delete entry with key from map. +/// +/// The delete map element helper call is used to delete values from maps. +unsafe fn raw_map_delete_elem(map: *mut c_void, key: *const c_void) -> i64 { + let map = Arc::from_raw(map as *const BpfMap); + let key_size = map.key_size(); + let key = core::slice::from_raw_parts(key as *const u8, key_size); + let res = map_delete_elem(&map, key); + let _ = Arc::into_raw(map); + match res { + Ok(_) => 0, + Err(e) => e as i64, + } +} + +pub fn map_delete_elem(map: &Arc, key: &[u8]) -> Result<()> { + let mut binding = map.inner_map().lock(); + let value = binding.delete_elem(key); + value +} + +/// For each element in map, call callback_fn function with map, callback_ctx and other map-specific +/// parameters. The callback_fn should be a static function and the callback_ctx should be a pointer +/// to the stack. The flags is used to control certain aspects of the helper. Currently, the flags must +/// be 0. +/// +/// The following are a list of supported map types and their respective expected callback signatures: +/// - BPF_MAP_TYPE_HASH +/// - BPF_MAP_TYPE_PERCPU_HASH +/// - BPF_MAP_TYPE_LRU_HASH +/// - BPF_MAP_TYPE_LRU_PERCPU_HASH +/// - BPF_MAP_TYPE_ARRAY +/// - BPF_MAP_TYPE_PERCPU_ARRAY +/// +/// `long (*callback_fn)(struct bpf_map *map, const void key, void *value, void *ctx);` +/// +/// For per_cpu maps, the map_value is the value on the cpu where the bpf_prog is running. +unsafe fn raw_map_for_each_elem( + map: *mut c_void, + cb: *const c_void, + ctx: *const c_void, + flags: u64, +) -> i64 { + let map = Arc::from_raw(map as *const BpfMap); + let cb = *core::mem::transmute::<*const c_void, *const BpfCallBackFn>(cb); + let res = map_for_each_elem(&map, cb, ctx as _, flags); + let _ = Arc::into_raw(map); + match res { + Ok(v) => v as i64, + Err(e) => e as i64, + } +} + +pub fn map_for_each_elem( + map: &Arc, + cb: BpfCallBackFn, + ctx: *const u8, + flags: u64, +) -> Result { + let mut binding = map.inner_map().lock(); + let value = binding.for_each_elem(cb, ctx, flags); + value +} + +/// Perform a lookup in percpu map for an entry associated to key on cpu. +/// +/// See https://ebpf-docs.dylanreimerink.nl/linux/helper-function/bpf_map_lookup_percpu_elem/ +unsafe fn raw_map_lookup_percpu_elem( + map: *mut c_void, + key: *const c_void, + cpu: u32, +) -> *const c_void { + let map = Arc::from_raw(map as *const BpfMap); + let key_size = map.key_size(); + let key = core::slice::from_raw_parts(key as *const u8, key_size); + let value = map_lookup_percpu_elem(&map, key, cpu); + // warning: We need to keep the map alive, so we don't drop it here. + let _ = Arc::into_raw(map); + match value { + Ok(Some(value)) => value as *const c_void, + _ => core::ptr::null_mut(), + } +} + +pub fn map_lookup_percpu_elem( + map: &Arc, + key: &[u8], + cpu: u32, +) -> Result> { + let mut binding = map.inner_map().lock(); + let value = binding.lookup_percpu_elem(key, cpu); + match value { + Ok(Some(value)) => Ok(Some(value.as_ptr())), + _ => Ok(None), + } +} +/// Push an element value in map. +/// +/// See https://ebpf-docs.dylanreimerink.nl/linux/helper-function/bpf_map_push_elem/ +unsafe fn raw_map_push_elem(map: *mut c_void, value: *const c_void, flags: u64) -> i64 { + let map = Arc::from_raw(map as *const BpfMap); + let value_size = map.value_size(); + let value = core::slice::from_raw_parts(value as *const u8, value_size); + let res = map_push_elem(&map, value, flags); + let _ = Arc::into_raw(map); + match res { + Ok(_) => 0, + Err(e) => e as i64, + } +} + +pub fn map_push_elem(map: &Arc, value: &[u8], flags: u64) -> Result<()> { + let mut binding = map.inner_map().lock(); + let value = binding.push_elem(value, flags); + value +} + +/// Pop an element from map. +/// +/// See https://ebpf-docs.dylanreimerink.nl/linux/helper-function/bpf_map_pop_elem/ +unsafe fn raw_map_pop_elem(map: *mut c_void, value: *mut c_void) -> i64 { + let map = Arc::from_raw(map as *const BpfMap); + let value_size = map.value_size(); + let value = core::slice::from_raw_parts_mut(value as *mut u8, value_size); + let res = map_pop_elem(&map, value); + let _ = Arc::into_raw(map); + match res { + Ok(_) => 0, + Err(e) => e as i64, + } +} + +pub fn map_pop_elem(map: &Arc, value: &mut [u8]) -> Result<()> { + let mut binding = map.inner_map().lock(); + let value = binding.pop_elem(value); + value +} + +/// Get an element from map without removing it. +/// +/// See https://ebpf-docs.dylanreimerink.nl/linux/helper-function/bpf_map_peek_elem/ +unsafe fn raw_map_peek_elem(map: *mut c_void, value: *mut c_void) -> i64 { + let map = Arc::from_raw(map as *const BpfMap); + let value_size = map.value_size(); + let value = core::slice::from_raw_parts_mut(value as *mut u8, value_size); + let res = map_peek_elem(&map, value); + let _ = Arc::into_raw(map); + match res { + Ok(_) => 0, + Err(e) => e as i64, + } +} + +pub fn map_peek_elem(map: &Arc, value: &mut [u8]) -> Result<()> { + let binding = map.inner_map().lock(); + let value = binding.peek_elem(value); + value +} + +pub static BPF_HELPER_FUN_SET: Lazy> = Lazy::new(); + +/// Initialize the helper functions. +pub fn init_helper_functions() { + use consts::*; + let mut map = BTreeMap::new(); + unsafe { + // Map helpers::Generic map helpers + map.insert(HELPER_MAP_LOOKUP_ELEM, define_func!(raw_map_lookup_elem)); + map.insert(HELPER_MAP_UPDATE_ELEM, define_func!(raw_map_update_elem)); + map.insert(HELPER_MAP_DELETE_ELEM, define_func!(raw_map_delete_elem)); + map.insert( + HELPER_MAP_FOR_EACH_ELEM, + define_func!(raw_map_for_each_elem), + ); + map.insert( + HELPER_MAP_LOOKUP_PERCPU_ELEM, + define_func!(raw_map_lookup_percpu_elem), + ); + // map.insert(93,define_func!(raw_bpf_spin_lock); + // map.insert(94,define_func!(raw_bpf_spin_unlock); + // Map helpers::Perf event array helpers + map.insert( + HELPER_PERF_EVENT_OUTPUT, + define_func!(raw_perf_event_output), + ); + // Probe and trace helpers::Memory helpers + map.insert(HELPER_BPF_PROBE_READ, define_func!(raw_bpf_probe_read)); + // Print helpers + map.insert(HELPER_TRACE_PRINTF, define_func!(trace_printf)); + + // Map helpers::Queue and stack helpers + map.insert(HELPER_MAP_PUSH_ELEM, define_func!(raw_map_push_elem)); + map.insert(HELPER_MAP_POP_ELEM, define_func!(raw_map_pop_elem)); + map.insert(HELPER_MAP_PEEK_ELEM, define_func!(raw_map_peek_elem)); + } + BPF_HELPER_FUN_SET.init(map); +} diff --git a/kernel/src/bpf/helper/print.rs b/kernel/src/bpf/helper/print.rs new file mode 100644 index 000000000..1e6c4e733 --- /dev/null +++ b/kernel/src/bpf/helper/print.rs @@ -0,0 +1,25 @@ +use core::{ + ffi::{c_char, c_int}, + fmt::Write, +}; + +use printf_compat::{format, output}; + +/// Printf according to the format string, function will return the number of bytes written(including '\0') +pub unsafe extern "C" fn printf(w: &mut impl Write, str: *const c_char, mut args: ...) -> c_int { + let bytes_written = format(str as _, args.as_va_list(), output::fmt_write(w)); + bytes_written + 1 +} + +struct TerminalOut; +impl Write for TerminalOut { + fn write_str(&mut self, s: &str) -> core::fmt::Result { + print!("{}", s); + Ok(()) + } +} + +/// See https://ebpf-docs.dylanreimerink.nl/linux/helper-function/bpf_trace_printk/ +pub fn trace_printf(fmt_ptr: u64, _fmt_len: u64, arg3: u64, arg4: u64, arg5: u64) -> u64 { + unsafe { printf(&mut TerminalOut, fmt_ptr as _, arg3, arg4, arg5) as u64 } +} diff --git a/kernel/src/bpf/map/array_map.rs b/kernel/src/bpf/map/array_map.rs new file mode 100644 index 000000000..a5b48d648 --- /dev/null +++ b/kernel/src/bpf/map/array_map.rs @@ -0,0 +1,283 @@ +//! BPF_MAP_TYPE_ARRAY and BPF_MAP_TYPE_PERCPU_ARRAY +//! +//! +//! See https://docs.kernel.org/bpf/map_array.html + +use super::super::Result; +use crate::bpf::map::util::round_up; +use crate::bpf::map::{BpfCallBackFn, BpfMapCommonOps, BpfMapMeta}; +use crate::mm::percpu::{PerCpu, PerCpuVar}; +use crate::smp::cpu::{smp_cpu_manager, ProcessorId}; +use alloc::{vec, vec::Vec}; +use core::{ + fmt::{Debug, Formatter}, + ops::{Index, IndexMut}, +}; +use log::info; +use system_error::SystemError; + +/// The array map type is a generic map type with no restrictions on the structure of the value. +/// Like a normal array, the array map has a numeric key starting at 0 and incrementing. +/// +/// See https://ebpf-docs.dylanreimerink.nl/linux/map-type/BPF_MAP_TYPE_ARRAY/ +#[derive(Debug)] +pub struct ArrayMap { + max_entries: u32, + data: ArrayMapData, +} + +struct ArrayMapData { + elem_size: u32, + /// The data is stored in a Vec with the size of elem_size * max_entries. + data: Vec, +} + +impl Debug for ArrayMapData { + fn fmt(&self, f: &mut Formatter<'_>) -> core::fmt::Result { + f.debug_struct("ArrayMapData") + .field("elem_size", &self.elem_size) + .field("data_len", &self.data.len()) + .finish() + } +} + +impl ArrayMapData { + pub fn new(elem_size: u32, max_entries: u32) -> Self { + debug_assert!(elem_size % 8 == 0); + let total_size = elem_size * max_entries; + let data = vec![0; total_size as usize]; + ArrayMapData { elem_size, data } + } +} + +impl Index for ArrayMapData { + type Output = [u8]; + fn index(&self, index: u32) -> &Self::Output { + let start = index * self.elem_size; + &self.data[start as usize..(start + self.elem_size) as usize] + } +} + +impl IndexMut for ArrayMapData { + fn index_mut(&mut self, index: u32) -> &mut Self::Output { + let start = index * self.elem_size; + &mut self.data[start as usize..(start + self.elem_size) as usize] + } +} + +impl ArrayMap { + pub fn new(attr: &BpfMapMeta) -> Result { + if attr.value_size == 0 || attr.max_entries == 0 || attr.key_size != 4 { + return Err(SystemError::EINVAL); + } + let elem_size = round_up(attr.value_size as usize, 8); + let data = ArrayMapData::new(elem_size as u32, attr.max_entries); + Ok(ArrayMap { + max_entries: attr.max_entries, + data, + }) + } +} + +impl BpfMapCommonOps for ArrayMap { + fn lookup_elem(&mut self, key: &[u8]) -> Result> { + if key.len() != 4 { + return Err(SystemError::EINVAL); + } + let index = u32::from_ne_bytes(key.try_into().map_err(|_| SystemError::EINVAL)?); + if index >= self.max_entries { + return Err(SystemError::EINVAL); + } + let val = self.data.index(index); + Ok(Some(val)) + } + fn update_elem(&mut self, key: &[u8], value: &[u8], _flags: u64) -> Result<()> { + if key.len() != 4 { + return Err(SystemError::EINVAL); + } + let index = u32::from_ne_bytes(key.try_into().map_err(|_| SystemError::EINVAL)?); + if index >= self.max_entries { + return Err(SystemError::EINVAL); + } + if value.len() > self.data.elem_size as usize { + return Err(SystemError::EINVAL); + } + let old_value = self.data.index_mut(index); + old_value[..value.len()].copy_from_slice(value); + Ok(()) + } + /// For ArrayMap, delete_elem is not supported. + fn delete_elem(&mut self, _key: &[u8]) -> Result<()> { + Err(SystemError::EINVAL) + } + fn for_each_elem(&mut self, cb: BpfCallBackFn, ctx: *const u8, flags: u64) -> Result { + if flags != 0 { + return Err(SystemError::EINVAL); + } + let mut total_used = 0; + for i in 0..self.max_entries { + let key = i.to_ne_bytes(); + let value = self.data.index(i); + total_used += 1; + let res = cb(&key, value, ctx); + // return value: 0 - continue, 1 - stop and return + if res != 0 { + break; + } + } + Ok(total_used) + } + + fn lookup_and_delete_elem(&mut self, _key: &[u8], _value: &mut [u8]) -> Result<()> { + Err(SystemError::EINVAL) + } + + fn get_next_key(&self, key: Option<&[u8]>, next_key: &mut [u8]) -> Result<()> { + if let Some(key) = key { + if key.len() != 4 { + return Err(SystemError::EINVAL); + } + let index = u32::from_ne_bytes(key.try_into().map_err(|_| SystemError::EINVAL)?); + if index == self.max_entries - 1 { + return Err(SystemError::ENOENT); + } + let next_index = index + 1; + next_key.copy_from_slice(&next_index.to_ne_bytes()); + } else { + next_key.copy_from_slice(&0u32.to_ne_bytes()); + } + Ok(()) + } + + fn freeze(&self) -> Result<()> { + info!("fake freeze done for ArrayMap"); + Ok(()) + } + fn first_value_ptr(&self) -> Result<*const u8> { + Ok(self.data.data.as_ptr()) + } +} + +/// This is the per-CPU variant of the [ArrayMap] map type. +/// +/// See https://ebpf-docs.dylanreimerink.nl/linux/map-type/BPF_MAP_TYPE_PERCPU_ARRAY/ +pub struct PerCpuArrayMap { + per_cpu_data: PerCpuVar, +} + +impl Debug for PerCpuArrayMap { + fn fmt(&self, f: &mut Formatter<'_>) -> core::fmt::Result { + f.debug_struct("PerCpuArrayMap") + .field("data", &self.per_cpu_data) + .finish() + } +} + +impl PerCpuArrayMap { + pub fn new(attr: &BpfMapMeta) -> Result { + let num_cpus = PerCpu::MAX_CPU_NUM; + let mut data = Vec::with_capacity(num_cpus as usize); + for _ in 0..num_cpus { + let array_map = ArrayMap::new(attr)?; + data.push(array_map); + } + let per_cpu_data = PerCpuVar::new(data).ok_or(SystemError::EINVAL)?; + Ok(PerCpuArrayMap { per_cpu_data }) + } +} + +impl BpfMapCommonOps for PerCpuArrayMap { + fn lookup_elem(&mut self, key: &[u8]) -> Result> { + self.per_cpu_data.get_mut().lookup_elem(key) + } + fn update_elem(&mut self, key: &[u8], value: &[u8], flags: u64) -> Result<()> { + self.per_cpu_data.get_mut().update_elem(key, value, flags) + } + fn delete_elem(&mut self, key: &[u8]) -> Result<()> { + self.per_cpu_data.get_mut().delete_elem(key) + } + fn for_each_elem(&mut self, cb: BpfCallBackFn, ctx: *const u8, flags: u64) -> Result { + self.per_cpu_data.get_mut().for_each_elem(cb, ctx, flags) + } + fn lookup_and_delete_elem(&mut self, _key: &[u8], _value: &mut [u8]) -> Result<()> { + Err(SystemError::EINVAL) + } + fn lookup_percpu_elem(&mut self, key: &[u8], cpu: u32) -> Result> { + unsafe { + self.per_cpu_data + .force_get_mut(ProcessorId::new(cpu)) + .lookup_elem(key) + } + } + fn get_next_key(&self, key: Option<&[u8]>, next_key: &mut [u8]) -> Result<()> { + self.per_cpu_data.get_mut().get_next_key(key, next_key) + } + fn first_value_ptr(&self) -> Result<*const u8> { + self.per_cpu_data.get_mut().first_value_ptr() + } +} + +/// See https://ebpf-docs.dylanreimerink.nl/linux/map-type/BPF_MAP_TYPE_PERF_EVENT_ARRAY/ +pub struct PerfEventArrayMap { + // The value is the file descriptor of the perf event. + fds: ArrayMapData, +} + +impl Debug for PerfEventArrayMap { + fn fmt(&self, f: &mut Formatter<'_>) -> core::fmt::Result { + f.debug_struct("PerfEventArrayMap") + .field("fds", &self.fds) + .finish() + } +} + +impl PerfEventArrayMap { + pub fn new(attr: &BpfMapMeta) -> Result { + let num_cpus = smp_cpu_manager().possible_cpus_count(); + if attr.key_size != 4 || attr.value_size != 4 || attr.max_entries != num_cpus { + return Err(SystemError::EINVAL); + } + let fds = ArrayMapData::new(4, num_cpus); + Ok(PerfEventArrayMap { fds }) + } +} + +impl BpfMapCommonOps for PerfEventArrayMap { + fn lookup_elem(&mut self, key: &[u8]) -> Result> { + let cpu_id = u32::from_ne_bytes(key.try_into().map_err(|_| SystemError::EINVAL)?); + let value = self.fds.index(cpu_id); + Ok(Some(value)) + } + fn update_elem(&mut self, key: &[u8], value: &[u8], _flags: u64) -> Result<()> { + assert_eq!(value.len(), 4); + let cpu_id = u32::from_ne_bytes(key.try_into().map_err(|_| SystemError::EINVAL)?); + let old_value = self.fds.index_mut(cpu_id); + old_value.copy_from_slice(value); + Ok(()) + } + fn delete_elem(&mut self, key: &[u8]) -> Result<()> { + let cpu_id = u32::from_ne_bytes(key.try_into().map_err(|_| SystemError::EINVAL)?); + self.fds.index_mut(cpu_id).copy_from_slice(&[0; 4]); + Ok(()) + } + fn for_each_elem(&mut self, cb: BpfCallBackFn, ctx: *const u8, _flags: u64) -> Result { + let mut total_used = 0; + let num_cpus = smp_cpu_manager().possible_cpus_count(); + for i in 0..num_cpus { + let key = i.to_ne_bytes(); + let value = self.fds.index(i); + total_used += 1; + let res = cb(&key, value, ctx); + if res != 0 { + break; + } + } + Ok(total_used) + } + fn lookup_and_delete_elem(&mut self, _key: &[u8], _value: &mut [u8]) -> Result<()> { + Err(SystemError::EINVAL) + } + fn first_value_ptr(&self) -> Result<*const u8> { + Ok(self.fds.data.as_ptr()) + } +} diff --git a/kernel/src/bpf/map/hash_map.rs b/kernel/src/bpf/map/hash_map.rs new file mode 100644 index 000000000..f3a740ba1 --- /dev/null +++ b/kernel/src/bpf/map/hash_map.rs @@ -0,0 +1,156 @@ +use super::Result; +use crate::bpf::map::util::{round_up, BpfMapUpdateElemFlags}; +use crate::bpf::map::{BpfCallBackFn, BpfMapCommonOps, BpfMapMeta}; +use crate::mm::percpu::{PerCpu, PerCpuVar}; +use crate::smp::cpu::ProcessorId; +use alloc::{collections::BTreeMap, vec::Vec}; +use core::fmt::Debug; +use system_error::SystemError; + +type BpfHashMapKey = Vec; +type BpfHashMapValue = Vec; + +/// The hash map type is a generic map type with no restrictions on the structure of the key and value. +/// Hash-maps are implemented using a hash table, allowing for lookups with arbitrary keys. +/// +/// See https://ebpf-docs.dylanreimerink.nl/linux/map-type/BPF_MAP_TYPE_HASH/ +#[derive(Debug)] +pub struct BpfHashMap { + _max_entries: u32, + _key_size: u32, + _value_size: u32, + data: BTreeMap, +} + +impl BpfHashMap { + pub fn new(attr: &BpfMapMeta) -> Result { + if attr.value_size == 0 || attr.max_entries == 0 { + return Err(SystemError::EINVAL); + } + let value_size = round_up(attr.value_size as usize, 8); + Ok(Self { + _max_entries: attr.max_entries, + _key_size: attr.key_size, + _value_size: value_size as u32, + data: BTreeMap::new(), + }) + } +} + +impl BpfMapCommonOps for BpfHashMap { + fn lookup_elem(&mut self, key: &[u8]) -> Result> { + let value = self.data.get(key).map(|v| v.as_slice()); + Ok(value) + } + fn update_elem(&mut self, key: &[u8], value: &[u8], flags: u64) -> Result<()> { + let _flags = BpfMapUpdateElemFlags::from_bits_truncate(flags); + self.data.insert(key.to_vec(), value.to_vec()); + Ok(()) + } + fn delete_elem(&mut self, key: &[u8]) -> Result<()> { + self.data.remove(key); + Ok(()) + } + fn for_each_elem(&mut self, cb: BpfCallBackFn, ctx: *const u8, flags: u64) -> Result { + if flags != 0 { + return Err(SystemError::EINVAL); + } + let mut total_used = 0; + for (key, value) in self.data.iter() { + let res = cb(key, value, ctx); + // return value: 0 - continue, 1 - stop and return + if res != 0 { + break; + } + total_used += 1; + } + Ok(total_used) + } + fn lookup_and_delete_elem(&mut self, key: &[u8], value: &mut [u8]) -> Result<()> { + let v = self + .data + .get(key) + .map(|v| v.as_slice()) + .ok_or(SystemError::ENOENT)?; + value.copy_from_slice(v); + self.data.remove(key); + Ok(()) + } + fn get_next_key(&self, key: Option<&[u8]>, next_key: &mut [u8]) -> Result<()> { + let mut iter = self.data.iter(); + if let Some(key) = key { + for (k, _) in iter.by_ref() { + if k.as_slice() == key { + break; + } + } + } + let res = iter.next(); + match res { + Some((k, _)) => { + next_key.copy_from_slice(k.as_slice()); + Ok(()) + } + None => Err(SystemError::ENOENT), + } + } +} + +/// This is the per-CPU variant of the [BpfHashMap] map type. +/// +/// See https://ebpf-docs.dylanreimerink.nl/linux/map-type/BPF_MAP_TYPE_PERCPU_HASH/ +pub struct PerCpuHashMap { + per_cpu_maps: PerCpuVar, +} + +impl Debug for PerCpuHashMap { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.debug_struct("PerCpuHashMap") + .field("maps", &self.per_cpu_maps) + .finish() + } +} +impl PerCpuHashMap { + pub fn new(attr: &BpfMapMeta) -> Result { + let num_cpus = PerCpu::MAX_CPU_NUM; + let mut data = Vec::with_capacity(num_cpus as usize); + for _ in 0..num_cpus { + let array_map = BpfHashMap::new(attr)?; + data.push(array_map); + } + let per_cpu_maps = PerCpuVar::new(data).ok_or(SystemError::EINVAL)?; + Ok(PerCpuHashMap { per_cpu_maps }) + } +} +impl BpfMapCommonOps for PerCpuHashMap { + fn lookup_elem(&mut self, key: &[u8]) -> Result> { + self.per_cpu_maps.get_mut().lookup_elem(key) + } + fn update_elem(&mut self, key: &[u8], value: &[u8], flags: u64) -> Result<()> { + self.per_cpu_maps.get_mut().update_elem(key, value, flags) + } + fn delete_elem(&mut self, key: &[u8]) -> Result<()> { + self.per_cpu_maps.get_mut().delete_elem(key) + } + fn for_each_elem(&mut self, cb: BpfCallBackFn, ctx: *const u8, flags: u64) -> Result { + self.per_cpu_maps.get_mut().for_each_elem(cb, ctx, flags) + } + fn lookup_and_delete_elem(&mut self, key: &[u8], value: &mut [u8]) -> Result<()> { + self.per_cpu_maps + .get_mut() + .lookup_and_delete_elem(key, value) + } + fn lookup_percpu_elem(&mut self, key: &[u8], cpu: u32) -> Result> { + unsafe { + self.per_cpu_maps + .force_get_mut(ProcessorId::new(cpu)) + .lookup_elem(key) + } + } + fn get_next_key(&self, key: Option<&[u8]>, next_key: &mut [u8]) -> Result<()> { + self.per_cpu_maps.get_mut().get_next_key(key, next_key) + } + fn first_value_ptr(&self) -> Result<*const u8> { + self.per_cpu_maps.get_mut().first_value_ptr() + } +} diff --git a/kernel/src/bpf/map/lru.rs b/kernel/src/bpf/map/lru.rs new file mode 100644 index 000000000..d4927aa4d --- /dev/null +++ b/kernel/src/bpf/map/lru.rs @@ -0,0 +1,151 @@ +use super::{BpfCallBackFn, BpfMapCommonOps, Result}; +use crate::bpf::map::util::BpfMapMeta; +use crate::mm::percpu::{PerCpu, PerCpuVar}; +use crate::smp::cpu::ProcessorId; +use alloc::vec::Vec; +use core::fmt::Debug; +use core::num::NonZero; +use lru::LruCache; +use system_error::SystemError; + +type BpfHashMapKey = Vec; +type BpfHashMapValue = Vec; +/// This map is the LRU (Least Recently Used) variant of the BPF_MAP_TYPE_HASH. +/// It is a generic map type that stores a fixed maximum number of key/value pairs. +/// When the map starts to get at capacity, the approximately least recently +/// used elements is removed to make room for new elements. +/// +/// See https://docs.ebpf.io/linux/map-type/BPF_MAP_TYPE_LRU_HASH/ +#[derive(Debug)] +pub struct LruMap { + _max_entries: u32, + data: LruCache, +} + +impl LruMap { + pub fn new(attr: &BpfMapMeta) -> Result { + if attr.value_size == 0 || attr.max_entries == 0 { + return Err(SystemError::EINVAL); + } + Ok(Self { + _max_entries: attr.max_entries, + data: LruCache::new( + NonZero::new(attr.max_entries as usize).ok_or(SystemError::EINVAL)?, + ), + }) + } +} + +impl BpfMapCommonOps for LruMap { + fn lookup_elem(&mut self, key: &[u8]) -> Result> { + let value = self.data.get(key).map(|v| v.as_slice()); + Ok(value) + } + fn update_elem(&mut self, key: &[u8], value: &[u8], _flags: u64) -> Result<()> { + self.data.put(key.to_vec(), value.to_vec()); + Ok(()) + } + fn delete_elem(&mut self, key: &[u8]) -> Result<()> { + self.data.pop(key); + Ok(()) + } + fn for_each_elem(&mut self, cb: BpfCallBackFn, ctx: *const u8, flags: u64) -> Result { + if flags != 0 { + return Err(SystemError::EINVAL); + } + let mut total_used = 0; + for (key, value) in self.data.iter() { + let res = cb(key, value, ctx); + // return value: 0 - continue, 1 - stop and return + if res != 0 { + break; + } + total_used += 1; + } + Ok(total_used) + } + fn lookup_and_delete_elem(&mut self, key: &[u8], value: &mut [u8]) -> Result<()> { + let v = self + .data + .get(key) + .map(|v| v.as_slice()) + .ok_or(SystemError::ENOENT)?; + value.copy_from_slice(v); + self.data.pop(key); + Ok(()) + } + fn get_next_key(&self, key: Option<&[u8]>, next_key: &mut [u8]) -> Result<()> { + let mut iter = self.data.iter(); + if let Some(key) = key { + for (k, _) in iter.by_ref() { + if k.as_slice() == key { + break; + } + } + } + let res = iter.next(); + match res { + Some((k, _)) => { + next_key.copy_from_slice(k.as_slice()); + Ok(()) + } + None => Err(SystemError::ENOENT), + } + } +} + +/// See https://ebpf-docs.dylanreimerink.nl/linux/map-type/BPF_MAP_TYPE_LRU_PERCPU_HASH/ +pub struct PerCpuLruMap { + per_cpu_maps: PerCpuVar, +} + +impl Debug for PerCpuLruMap { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.debug_struct("PerCpuLruMap") + .field("maps", &self.per_cpu_maps) + .finish() + } +} + +impl PerCpuLruMap { + pub fn new(attr: &BpfMapMeta) -> Result { + let num_cpus = PerCpu::MAX_CPU_NUM; + let mut data = Vec::with_capacity(num_cpus as usize); + for _ in 0..num_cpus { + let array_map = LruMap::new(attr)?; + data.push(array_map); + } + let per_cpu_maps = PerCpuVar::new(data).ok_or(SystemError::EINVAL)?; + Ok(PerCpuLruMap { per_cpu_maps }) + } +} + +impl BpfMapCommonOps for PerCpuLruMap { + fn lookup_elem(&mut self, key: &[u8]) -> Result> { + self.per_cpu_maps.get_mut().lookup_elem(key) + } + fn update_elem(&mut self, key: &[u8], value: &[u8], flags: u64) -> Result<()> { + self.per_cpu_maps.get_mut().update_elem(key, value, flags) + } + fn delete_elem(&mut self, key: &[u8]) -> Result<()> { + self.per_cpu_maps.get_mut().delete_elem(key) + } + fn for_each_elem(&mut self, cb: BpfCallBackFn, ctx: *const u8, flags: u64) -> Result { + self.per_cpu_maps.get_mut().for_each_elem(cb, ctx, flags) + } + fn lookup_and_delete_elem(&mut self, key: &[u8], value: &mut [u8]) -> Result<()> { + self.per_cpu_maps + .get_mut() + .lookup_and_delete_elem(key, value) + } + fn lookup_percpu_elem(&mut self, key: &[u8], cpu: u32) -> Result> { + unsafe { + self.per_cpu_maps + .force_get_mut(ProcessorId::new(cpu)) + .lookup_elem(key) + } + } + fn get_next_key(&self, key: Option<&[u8]>, next_key: &mut [u8]) -> Result<()> { + self.per_cpu_maps.get_mut().get_next_key(key, next_key) + } +} diff --git a/kernel/src/bpf/map/mod.rs b/kernel/src/bpf/map/mod.rs new file mode 100644 index 000000000..cc8eeffef --- /dev/null +++ b/kernel/src/bpf/map/mod.rs @@ -0,0 +1,416 @@ +mod array_map; +mod hash_map; +mod lru; +mod queue; +mod util; + +use super::Result; +use crate::bpf::map::array_map::{ArrayMap, PerCpuArrayMap, PerfEventArrayMap}; +use crate::bpf::map::hash_map::PerCpuHashMap; +use crate::bpf::map::util::{BpfMapGetNextKeyArg, BpfMapMeta, BpfMapUpdateArg}; +use crate::filesystem::vfs::file::{File, FileMode}; +use crate::filesystem::vfs::syscall::ModeType; +use crate::filesystem::vfs::{FilePrivateData, FileSystem, FileType, IndexNode, Metadata}; +use crate::include::bindings::linux_bpf::{bpf_attr, bpf_map_type}; +use crate::libs::casting::DowncastArc; +use crate::libs::spinlock::{SpinLock, SpinLockGuard}; +use crate::process::ProcessManager; +use crate::syscall::user_access::{UserBufferReader, UserBufferWriter}; +use alloc::boxed::Box; +use alloc::string::String; +use alloc::sync::Arc; +use alloc::vec::Vec; +use core::any::Any; +use core::fmt::Debug; +use intertrait::CastFromSync; +use log::{error, info}; +use system_error::SystemError; + +#[derive(Debug)] +pub struct BpfMap { + inner_map: SpinLock>, + meta: BpfMapMeta, +} + +pub type BpfCallBackFn = fn(key: &[u8], value: &[u8], ctx: *const u8) -> i32; + +pub trait BpfMapCommonOps: Send + Sync + Debug + CastFromSync { + /// Lookup an element in the map. + /// + /// See https://ebpf-docs.dylanreimerink.nl/linux/helper-function/bpf_map_lookup_elem/ + fn lookup_elem(&mut self, _key: &[u8]) -> Result> { + Err(SystemError::ENOSYS) + } + /// Update an element in the map. + /// + /// See https://ebpf-docs.dylanreimerink.nl/linux/helper-function/bpf_map_update_elem/ + fn update_elem(&mut self, _key: &[u8], _value: &[u8], _flags: u64) -> Result<()> { + Err(SystemError::ENOSYS) + } + /// Delete an element from the map. + /// + /// See https://ebpf-docs.dylanreimerink.nl/linux/helper-function/bpf_map_delete_elem/ + fn delete_elem(&mut self, _key: &[u8]) -> Result<()> { + Err(SystemError::ENOSYS) + } + /// For each element in map, call callback_fn function with map, + /// callback_ctx and other map-specific parameters. + /// + /// See https://ebpf-docs.dylanreimerink.nl/linux/helper-function/bpf_for_each_map_elem/ + fn for_each_elem(&mut self, _cb: BpfCallBackFn, _ctx: *const u8, _flags: u64) -> Result { + Err(SystemError::ENOSYS) + } + /// Look up an element with the given key in the map referred to by the file descriptor fd, + /// and if found, delete the element. + fn lookup_and_delete_elem(&mut self, _key: &[u8], _value: &mut [u8]) -> Result<()> { + Err(SystemError::ENOSYS) + } + + /// erform a lookup in percpu map for an entry associated to key on cpu. + fn lookup_percpu_elem(&mut self, _key: &[u8], _cpu: u32) -> Result> { + Err(SystemError::ENOSYS) + } + /// Get the next key in the map. If key is None, get the first key. + /// + /// Called from syscall + fn get_next_key(&self, _key: Option<&[u8]>, _next_key: &mut [u8]) -> Result<()> { + Err(SystemError::ENOSYS) + } + + /// Push an element value in map. + fn push_elem(&mut self, _value: &[u8], _flags: u64) -> Result<()> { + Err(SystemError::ENOSYS) + } + + /// Pop an element value from map. + fn pop_elem(&mut self, _value: &mut [u8]) -> Result<()> { + Err(SystemError::ENOSYS) + } + + /// Peek an element value from map. + fn peek_elem(&self, _value: &mut [u8]) -> Result<()> { + Err(SystemError::ENOSYS) + } + + /// Freeze the map. + /// + /// It's useful for .rodata maps. + fn freeze(&self) -> Result<()> { + Err(SystemError::ENOSYS) + } + + /// Get the first value pointer. + fn first_value_ptr(&self) -> Result<*const u8> { + Err(SystemError::ENOSYS) + } +} +impl DowncastArc for dyn BpfMapCommonOps { + fn as_any_arc(self: Arc) -> Arc { + self + } +} +impl BpfMap { + pub fn new(map: Box, meta: BpfMapMeta) -> Self { + assert_ne!(meta.key_size, 0); + BpfMap { + inner_map: SpinLock::new(map), + meta, + } + } + + pub fn inner_map(&self) -> &SpinLock> { + &self.inner_map + } + + pub fn key_size(&self) -> usize { + self.meta.key_size as usize + } + + pub fn value_size(&self) -> usize { + self.meta.value_size as usize + } +} + +impl IndexNode for BpfMap { + fn open(&self, _data: SpinLockGuard, _mode: &FileMode) -> Result<()> { + Ok(()) + } + fn close(&self, _data: SpinLockGuard) -> Result<()> { + Ok(()) + } + fn read_at( + &self, + _offset: usize, + _len: usize, + _buf: &mut [u8], + _data: SpinLockGuard, + ) -> Result { + Err(SystemError::ENOSYS) + } + + fn write_at( + &self, + _offset: usize, + _len: usize, + _buf: &[u8], + _data: SpinLockGuard, + ) -> Result { + Err(SystemError::ENOSYS) + } + + fn metadata(&self) -> Result { + let meta = Metadata { + mode: ModeType::from_bits_truncate(0o755), + file_type: FileType::File, + ..Default::default() + }; + Ok(meta) + } + + fn resize(&self, _len: usize) -> Result<()> { + Ok(()) + } + + fn fs(&self) -> Arc { + todo!("BpfMap does not have a filesystem") + } + + fn as_any_ref(&self) -> &dyn Any { + self + } + + fn list(&self) -> Result> { + Err(SystemError::ENOSYS) + } +} + +/// Create a map and return a file descriptor that refers to +/// the map. The close-on-exec file descriptor flag +/// is automatically enabled for the new file descriptor. +/// +/// See https://ebpf-docs.dylanreimerink.nl/linux/syscall/BPF_MAP_CREATE/ +pub fn bpf_map_create(attr: &bpf_attr) -> Result { + let map_meta = BpfMapMeta::try_from(attr)?; + info!("The map attr is {:#?}", map_meta); + let map: Box = match map_meta.map_type { + bpf_map_type::BPF_MAP_TYPE_ARRAY => { + let array_map = ArrayMap::new(&map_meta)?; + Box::new(array_map) + } + bpf_map_type::BPF_MAP_TYPE_PERCPU_ARRAY => { + let per_cpu_array_map = PerCpuArrayMap::new(&map_meta)?; + Box::new(per_cpu_array_map) + } + bpf_map_type::BPF_MAP_TYPE_PERF_EVENT_ARRAY => { + let perf_event_array_map = PerfEventArrayMap::new(&map_meta)?; + Box::new(perf_event_array_map) + } + + bpf_map_type::BPF_MAP_TYPE_CPUMAP + | bpf_map_type::BPF_MAP_TYPE_DEVMAP + | bpf_map_type::BPF_MAP_TYPE_DEVMAP_HASH => { + error!("bpf map type {:?} not implemented", map_meta.map_type); + Err(SystemError::EINVAL)? + } + bpf_map_type::BPF_MAP_TYPE_HASH => { + let hash_map = hash_map::BpfHashMap::new(&map_meta)?; + Box::new(hash_map) + } + bpf_map_type::BPF_MAP_TYPE_PERCPU_HASH => { + let per_cpu_hash_map = PerCpuHashMap::new(&map_meta)?; + Box::new(per_cpu_hash_map) + } + bpf_map_type::BPF_MAP_TYPE_QUEUE => { + let queue_map = queue::QueueMap::new(&map_meta)?; + Box::new(queue_map) + } + bpf_map_type::BPF_MAP_TYPE_STACK => { + let stack_map = queue::StackMap::new(&map_meta)?; + Box::new(stack_map) + } + bpf_map_type::BPF_MAP_TYPE_LRU_HASH => { + let lru_hash_map = lru::LruMap::new(&map_meta)?; + Box::new(lru_hash_map) + } + bpf_map_type::BPF_MAP_TYPE_LRU_PERCPU_HASH => { + let lru_per_cpu_hash_map = lru::PerCpuLruMap::new(&map_meta)?; + Box::new(lru_per_cpu_hash_map) + } + _ => { + unimplemented!("bpf map type {:?} not implemented", map_meta.map_type) + } + }; + let bpf_map = BpfMap::new(map, map_meta); + let fd_table = ProcessManager::current_pcb().fd_table(); + let file = File::new(Arc::new(bpf_map), FileMode::O_RDWR | FileMode::O_CLOEXEC)?; + let fd = fd_table.write().alloc_fd(file, None).map(|x| x as usize)?; + info!("create map with fd: [{}]", fd); + Ok(fd) +} + +/// Create or update an element (key/value pair) in a specified map. +/// +/// See https://ebpf-docs.dylanreimerink.nl/linux/syscall/BPF_MAP_UPDATE_ELEM/ +pub fn bpf_map_update_elem(attr: &bpf_attr) -> Result { + let arg = BpfMapUpdateArg::from(attr); + info!(": {:#x?}", arg); + let map = get_map_file(arg.map_fd as i32)?; + let meta = &map.meta; + let key_size = meta.key_size as usize; + let value_size = meta.value_size as usize; + + let key_buf = UserBufferReader::new(arg.key as *const u8, key_size, true)?; + let value_buf = UserBufferReader::new(arg.value as *const u8, value_size, true)?; + + let key = key_buf.read_from_user(0)?; + let value = value_buf.read_from_user(0)?; + map.inner_map.lock().update_elem(key, value, arg.flags)?; + info!("bpf_map_update_elem ok"); + Ok(0) +} + +pub fn bpf_map_freeze(attr: &bpf_attr) -> Result { + let arg = BpfMapUpdateArg::from(attr); + let map_fd = arg.map_fd; + info!(": map_fd: {:}", map_fd); + let map = get_map_file(map_fd as i32)?; + map.inner_map.lock().freeze()?; + Ok(0) +} + +/// Look up an element by key in a specified map and return its value. +/// +/// See https://ebpf-docs.dylanreimerink.nl/linux/syscall/BPF_MAP_LOOKUP_ELEM/ +pub fn bpf_lookup_elem(attr: &bpf_attr) -> Result { + let arg = BpfMapUpdateArg::from(attr); + // info!(": {:#x?}", arg); + let map = get_map_file(arg.map_fd as _)?; + let meta = &map.meta; + let key_size = meta.key_size as usize; + let value_size = meta.value_size as usize; + + let key_buf = UserBufferReader::new(arg.key as *const u8, key_size, true)?; + let mut value_buf = UserBufferWriter::new(arg.value as *mut u8, value_size, true)?; + + let key = key_buf.read_from_user(0)?; + + let mut inner = map.inner_map.lock(); + let r_value = inner.lookup_elem(key)?; + if let Some(r_value) = r_value { + value_buf.copy_to_user(r_value, 0)?; + Ok(0) + } else { + Err(SystemError::ENOENT) + } +} +/// Look up an element by key in a specified map and return the key of the next element. +/// +/// - If key is `None`, the operation returns zero and sets the next_key pointer to the key of the first element. +/// - If key is `Some(T)`, the operation returns zero and sets the next_key pointer to the key of the next element. +/// - If key is the last element, returns -1 and errno is set to ENOENT. +/// +/// See https://ebpf-docs.dylanreimerink.nl/linux/syscall/BPF_MAP_GET_NEXT_KEY/ +pub fn bpf_map_get_next_key(attr: &bpf_attr) -> Result { + let arg = BpfMapGetNextKeyArg::from(attr); + // info!(": {:#x?}", arg); + let map = get_map_file(arg.map_fd as i32)?; + let meta = &map.meta; + let key_size = meta.key_size as usize; + + let key = if let Some(key_ptr) = arg.key { + let key_buf = UserBufferReader::new(key_ptr as *const u8, key_size, true)?; + let key = key_buf.read_from_user(0)?.to_vec(); + Some(key) + } else { + None + }; + let key = key.as_deref(); + let mut next_key_buf = UserBufferWriter::new(arg.next_key as *mut u8, key_size, true)?; + let inner = map.inner_map.lock(); + let next_key = next_key_buf.buffer(0)?; + inner.get_next_key(key, next_key)?; + // info!("next_key: {:?}", next_key); + Ok(0) +} + +/// Look up and delete an element by key in a specified map. +/// +/// # WARN +/// +/// Not all map types (particularly array maps) support this operation, +/// instead a zero value can be written to the map value. Check the map types page to check for support. +/// +/// See https://ebpf-docs.dylanreimerink.nl/linux/syscall/BPF_MAP_DELETE_ELEM/ +pub fn bpf_map_delete_elem(attr: &bpf_attr) -> Result { + let arg = BpfMapUpdateArg::from(attr); + // info!(": {:#x?}", arg); + let map = get_map_file(arg.map_fd as i32)?; + let meta = &map.meta; + let key_size = meta.key_size as usize; + + let key_buf = UserBufferReader::new(arg.key as *const u8, key_size, true)?; + let key = key_buf.read_from_user(0)?; + map.inner_map.lock().delete_elem(key)?; + Ok(0) +} + +/// Iterate and fetch multiple elements in a map. +/// +/// See https://ebpf-docs.dylanreimerink.nl/linux/syscall/BPF_MAP_LOOKUP_BATCH/ +pub fn bpf_map_lookup_batch(_attr: &bpf_attr) -> Result { + todo!() +} + +/// Look up an element with the given key in the map referred to by the file descriptor fd, +/// and if found, delete the element. +/// +/// For BPF_MAP_TYPE_QUEUE and BPF_MAP_TYPE_STACK map types, the flags argument needs to be set to 0, +/// but for other map types, it may be specified as: +/// - BPF_F_LOCK : If this flag is set, the command will acquire the spin-lock of the map value we are looking up. +/// +/// If the map contains no spin-lock in its value, -EINVAL will be returned by the command. +/// +/// The BPF_MAP_TYPE_QUEUE and BPF_MAP_TYPE_STACK map types implement this command as a “pop” operation, +/// deleting the top element rather than one corresponding to key. +/// The key and key_len parameters should be zeroed when issuing this operation for these map types. +/// +/// This command is only valid for the following map types: +/// - BPF_MAP_TYPE_QUEUE +/// - BPF_MAP_TYPE_STACK +/// - BPF_MAP_TYPE_HASH +/// - BPF_MAP_TYPE_PERCPU_HASH +/// - BPF_MAP_TYPE_LRU_HASH +/// - BPF_MAP_TYPE_LRU_PERCPU_HASH +/// +/// +/// See https://ebpf-docs.dylanreimerink.nl/linux/syscall/BPF_MAP_LOOKUP_AND_DELETE_ELEM/ +pub fn bpf_map_lookup_and_delete_elem(attr: &bpf_attr) -> Result { + let arg = BpfMapUpdateArg::from(attr); + // info!(": {:#x?}", arg); + let map = get_map_file(arg.map_fd as i32)?; + let meta = &map.meta; + let key_size = meta.key_size as usize; + let value_size = meta.value_size as usize; + + let key_buf = UserBufferReader::new(arg.key as *const u8, key_size, true)?; + let mut value_buf = UserBufferWriter::new(arg.value as *mut u8, value_size, true)?; + + let value = value_buf.buffer(0)?; + let key = key_buf.read_from_user(0)?; + let mut inner = map.inner_map.lock(); + inner.lookup_and_delete_elem(key, value)?; + Ok(0) +} + +fn get_map_file(fd: i32) -> Result> { + let fd_table = ProcessManager::current_pcb().fd_table(); + let map = fd_table + .read() + .get_file_by_fd(fd) + .ok_or(SystemError::EBADF)?; + let map = map + .inode() + .downcast_arc::() + .ok_or(SystemError::EINVAL)?; + Ok(map) +} diff --git a/kernel/src/bpf/map/queue.rs b/kernel/src/bpf/map/queue.rs new file mode 100644 index 000000000..2a97edfff --- /dev/null +++ b/kernel/src/bpf/map/queue.rs @@ -0,0 +1,154 @@ +use super::{BpfMapCommonOps, Result}; +use crate::bpf::map::util::{BpfMapMeta, BpfMapUpdateElemFlags}; +use alloc::vec::Vec; +use core::fmt::Debug; +use core::ops::Deref; +use core::ops::DerefMut; +use system_error::SystemError; + +type BpfQueueValue = Vec; +/// BPF_MAP_TYPE_QUEUE provides FIFO storage and BPF_MAP_TYPE_STACK provides LIFO storage for BPF programs. +/// These maps support peek, pop and push operations that are exposed to BPF programs through the respective helpers. +/// These operations are exposed to userspace applications using the existing bpf syscall in the following way: +/// - `BPF_MAP_LOOKUP_ELEM` -> `peek` +/// - `BPF_MAP_UPDATE_ELEM` -> `push` +/// - `BPF_MAP_LOOKUP_AND_DELETE_ELEM ` -> `pop` +/// +/// See https://docs.kernel.org/bpf/map_queue_stack.html +pub trait SpecialMap: Debug + Send + Sync + 'static { + /// Returns the number of elements the queue can hold. + fn push(&mut self, value: BpfQueueValue, flags: BpfMapUpdateElemFlags) -> Result<()>; + /// Removes the first element and returns it. + fn pop(&mut self) -> Option; + /// Returns the first element without removing it. + fn peek(&self) -> Option<&BpfQueueValue>; +} + +/// The queue map type is a generic map type, resembling a FIFO (First-In First-Out) queue. +/// +/// This map type has no keys, only values. The size and type of the values can be specified by the user +/// to fit a large variety of use cases. The typical use-case for this map type is to keep track of +/// a pool of elements such as available network ports when implementing NAT (network address translation). +/// +/// As apposed to most map types, this map type uses a custom set of helpers to pop, peek and push elements. +/// +/// See https://ebpf-docs.dylanreimerink.nl/linux/map-type/BPF_MAP_TYPE_QUEUE/ +#[derive(Debug)] +pub struct QueueMap { + max_entries: u32, + data: Vec, +} + +impl QueueMap { + pub fn new(attr: &BpfMapMeta) -> Result { + if attr.value_size == 0 || attr.max_entries == 0 || attr.key_size != 0 { + return Err(SystemError::EINVAL); + } + let data = Vec::with_capacity(attr.max_entries as usize); + Ok(Self { + max_entries: attr.max_entries, + data, + }) + } +} + +impl SpecialMap for QueueMap { + fn push(&mut self, value: BpfQueueValue, flags: BpfMapUpdateElemFlags) -> Result<()> { + if self.data.len() == self.max_entries as usize { + if flags.contains(BpfMapUpdateElemFlags::BPF_EXIST) { + // remove the first element + self.data.remove(0); + } else { + return Err(SystemError::ENOSPC); + } + } + self.data.push(value); + Ok(()) + } + fn pop(&mut self) -> Option { + if self.data.is_empty() { + return None; + } + Some(self.data.remove(0)) + } + fn peek(&self) -> Option<&BpfQueueValue> { + self.data.first() + } +} +/// The stack map type is a generic map type, resembling a stack data structure. +/// +/// See https://ebpf-docs.dylanreimerink.nl/linux/map-type/BPF_MAP_TYPE_STACK/ +#[derive(Debug)] +pub struct StackMap(QueueMap); + +impl StackMap { + pub fn new(attr: &BpfMapMeta) -> Result { + QueueMap::new(attr).map(StackMap) + } +} + +impl Deref for StackMap { + type Target = QueueMap; + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl DerefMut for StackMap { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.0 + } +} + +impl SpecialMap for StackMap { + fn push(&mut self, value: BpfQueueValue, flags: BpfMapUpdateElemFlags) -> Result<()> { + if self.data.len() == self.max_entries as usize { + if flags.contains(BpfMapUpdateElemFlags::BPF_EXIST) { + // remove the last element + self.data.pop(); + } else { + return Err(SystemError::ENOSPC); + } + } + self.data.push(value); + Ok(()) + } + fn pop(&mut self) -> Option { + self.data.pop() + } + fn peek(&self) -> Option<&BpfQueueValue> { + self.data.last() + } +} + +impl BpfMapCommonOps for T { + /// Equal to [QueueMap::peek] + fn lookup_elem(&mut self, _key: &[u8]) -> Result> { + Ok(self.peek().map(|v| v.as_slice())) + } + /// Equal to [QueueMap::push] + fn update_elem(&mut self, _key: &[u8], value: &[u8], flags: u64) -> Result<()> { + let flag = BpfMapUpdateElemFlags::from_bits_truncate(flags); + self.push(value.to_vec(), flag) + } + /// Equal to [QueueMap::pop] + fn lookup_and_delete_elem(&mut self, _key: &[u8], value: &mut [u8]) -> Result<()> { + if let Some(v) = self.pop() { + value.copy_from_slice(&v); + Ok(()) + } else { + Err(SystemError::ENOENT) + } + } + fn push_elem(&mut self, value: &[u8], flags: u64) -> Result<()> { + self.update_elem(&[], value, flags) + } + fn pop_elem(&mut self, value: &mut [u8]) -> Result<()> { + self.lookup_and_delete_elem(&[], value) + } + fn peek_elem(&self, value: &mut [u8]) -> Result<()> { + self.peek() + .map(|v| value.copy_from_slice(v)) + .ok_or(SystemError::ENOENT) + } +} diff --git a/kernel/src/bpf/map/util.rs b/kernel/src/bpf/map/util.rs new file mode 100644 index 000000000..fdfc594ea --- /dev/null +++ b/kernel/src/bpf/map/util.rs @@ -0,0 +1,100 @@ +use crate::include::bindings::linux_bpf::{bpf_attr, bpf_map_type}; +use alloc::string::{String, ToString}; +use core::ffi::CStr; +use num_traits::FromPrimitive; +use system_error::SystemError; + +#[derive(Debug, Clone)] +pub struct BpfMapMeta { + pub map_type: bpf_map_type, + pub key_size: u32, + pub value_size: u32, + pub max_entries: u32, + pub _map_flags: u32, + pub _map_name: String, +} + +impl TryFrom<&bpf_attr> for BpfMapMeta { + type Error = SystemError; + fn try_from(value: &bpf_attr) -> Result { + let u = unsafe { &value.__bindgen_anon_1 }; + let map_name_slice = unsafe { + core::slice::from_raw_parts(u.map_name.as_ptr() as *const u8, u.map_name.len()) + }; + let map_name = CStr::from_bytes_until_nul(map_name_slice) + .map_err(|_| SystemError::EINVAL)? + .to_str() + .map_err(|_| SystemError::EINVAL)? + .to_string(); + let map_type = bpf_map_type::from_u32(u.map_type).ok_or(SystemError::EINVAL)?; + Ok(BpfMapMeta { + map_type, + key_size: u.key_size, + value_size: u.value_size, + max_entries: u.max_entries, + _map_flags: u.map_flags, + _map_name: map_name, + }) + } +} + +#[derive(Debug)] +pub struct BpfMapUpdateArg { + pub map_fd: u32, + pub key: u64, + pub value: u64, + pub flags: u64, +} + +impl From<&bpf_attr> for BpfMapUpdateArg { + fn from(value: &bpf_attr) -> Self { + unsafe { + let u = &value.__bindgen_anon_2; + BpfMapUpdateArg { + map_fd: u.map_fd, + key: u.key, + value: u.__bindgen_anon_1.value, + flags: u.flags, + } + } + } +} +#[derive(Debug)] +pub struct BpfMapGetNextKeyArg { + pub map_fd: u32, + pub key: Option, + pub next_key: u64, +} + +impl From<&bpf_attr> for BpfMapGetNextKeyArg { + fn from(value: &bpf_attr) -> Self { + unsafe { + let u = &value.__bindgen_anon_2; + BpfMapGetNextKeyArg { + map_fd: u.map_fd, + key: if u.key != 0 { Some(u.key) } else { None }, + next_key: u.__bindgen_anon_1.next_key, + } + } + } +} + +#[inline] +/// Round up `x` to the nearest multiple of `align`. +pub fn round_up(x: usize, align: usize) -> usize { + (x + align - 1) & !(align - 1) +} + +bitflags! { + /// flags for BPF_MAP_UPDATE_ELEM command + pub struct BpfMapUpdateElemFlags: u64 { + /// create new element or update existing + const BPF_ANY = 0; + /// create new element if it didn't exist + const BPF_NOEXIST = 1; + /// update existing element + const BPF_EXIST = 2; + /// spin_lock-ed map_lookup/map_update + const BPF_F_LOCK = 4; + } +} diff --git a/kernel/src/bpf/mod.rs b/kernel/src/bpf/mod.rs new file mode 100644 index 000000000..8e84f2055 --- /dev/null +++ b/kernel/src/bpf/mod.rs @@ -0,0 +1,50 @@ +pub mod helper; +pub mod map; +pub mod prog; +use crate::include::bindings::linux_bpf::{bpf_attr, bpf_cmd}; +use crate::syscall::user_access::UserBufferReader; +use crate::syscall::Syscall; +use log::error; +use num_traits::FromPrimitive; +use system_error::SystemError; + +type Result = core::result::Result; + +impl Syscall { + pub fn sys_bpf(cmd: u32, attr: *mut u8, size: u32) -> Result { + let buf = UserBufferReader::new(attr, size as usize, true)?; + let attr = buf.read_one_from_user::(0)?; + let cmd = bpf_cmd::from_u32(cmd).ok_or(SystemError::EINVAL)?; + bpf(cmd, attr) + } +} + +pub fn bpf(cmd: bpf_cmd, attr: &bpf_attr) -> Result { + let res = match cmd { + // Map related commands + bpf_cmd::BPF_MAP_CREATE => map::bpf_map_create(attr), + bpf_cmd::BPF_MAP_UPDATE_ELEM => map::bpf_map_update_elem(attr), + bpf_cmd::BPF_MAP_LOOKUP_ELEM => map::bpf_lookup_elem(attr), + bpf_cmd::BPF_MAP_GET_NEXT_KEY => map::bpf_map_get_next_key(attr), + bpf_cmd::BPF_MAP_DELETE_ELEM => map::bpf_map_delete_elem(attr), + bpf_cmd::BPF_MAP_LOOKUP_AND_DELETE_ELEM => map::bpf_map_lookup_and_delete_elem(attr), + bpf_cmd::BPF_MAP_LOOKUP_BATCH => map::bpf_map_lookup_batch(attr), + bpf_cmd::BPF_MAP_FREEZE => map::bpf_map_freeze(attr), + // Program related commands + bpf_cmd::BPF_PROG_LOAD => prog::bpf_prog_load(attr), + // Object creation commands + bpf_cmd::BPF_BTF_LOAD => { + error!("bpf cmd {:?} not implemented", cmd); + return Err(SystemError::ENOSYS); + } + ty => { + unimplemented!("bpf cmd {:?} not implemented", ty) + } + }; + res +} + +/// Initialize the BPF system +pub fn init_bpf_system() { + helper::init_helper_functions(); +} diff --git a/kernel/src/bpf/prog/mod.rs b/kernel/src/bpf/prog/mod.rs new file mode 100644 index 000000000..569af42fe --- /dev/null +++ b/kernel/src/bpf/prog/mod.rs @@ -0,0 +1,123 @@ +mod util; +mod verifier; + +use super::Result; +use crate::bpf::map::BpfMap; +use crate::bpf::prog::util::{BpfProgMeta, BpfProgVerifierInfo}; +use crate::bpf::prog::verifier::BpfProgVerifier; +use crate::filesystem::vfs::file::{File, FileMode}; +use crate::filesystem::vfs::syscall::ModeType; +use crate::filesystem::vfs::{FilePrivateData, FileSystem, FileType, IndexNode, Metadata}; +use crate::include::bindings::linux_bpf::bpf_attr; +use crate::libs::spinlock::SpinLockGuard; +use crate::process::ProcessManager; +use alloc::string::String; +use alloc::sync::Arc; +use alloc::vec::Vec; +use core::any::Any; +use system_error::SystemError; + +#[derive(Debug)] +pub struct BpfProg { + meta: BpfProgMeta, + raw_file_ptr: Vec, +} + +impl BpfProg { + pub fn new(meta: BpfProgMeta) -> Self { + Self { + meta, + raw_file_ptr: Vec::new(), + } + } + + pub fn insns(&self) -> &[u8] { + &self.meta.insns + } + + pub fn insns_mut(&mut self) -> &mut [u8] { + &mut self.meta.insns + } + + pub fn insert_map(&mut self, map_ptr: usize) { + self.raw_file_ptr.push(map_ptr); + } +} + +impl IndexNode for BpfProg { + fn open(&self, _data: SpinLockGuard, _mode: &FileMode) -> Result<()> { + Ok(()) + } + fn close(&self, _data: SpinLockGuard) -> Result<()> { + Ok(()) + } + fn read_at( + &self, + _offset: usize, + _len: usize, + _buf: &mut [u8], + _data: SpinLockGuard, + ) -> Result { + Err(SystemError::ENOSYS) + } + + fn write_at( + &self, + _offset: usize, + _len: usize, + _buf: &[u8], + _data: SpinLockGuard, + ) -> Result { + Err(SystemError::ENOSYS) + } + + fn metadata(&self) -> Result { + let meta = Metadata { + mode: ModeType::from_bits_truncate(0o755), + file_type: FileType::File, + ..Default::default() + }; + Ok(meta) + } + + fn resize(&self, _len: usize) -> Result<()> { + Ok(()) + } + + fn fs(&self) -> Arc { + panic!("BpfProg does not have a filesystem") + } + + fn as_any_ref(&self) -> &dyn Any { + self + } + + fn list(&self) -> Result> { + Err(SystemError::ENOSYS) + } +} + +impl Drop for BpfProg { + fn drop(&mut self) { + unsafe { + for ptr in self.raw_file_ptr.iter() { + let file = Arc::from_raw(*ptr as *const u8 as *const BpfMap); + drop(file) + } + } + } +} +/// Load a BPF program into the kernel. +/// +/// See https://ebpf-docs.dylanreimerink.nl/linux/syscall/BPF_PROG_LOAD/ +pub fn bpf_prog_load(attr: &bpf_attr) -> Result { + let args = BpfProgMeta::try_from(attr)?; + // info!("bpf_prog_load: {:#?}", args); + let log_info = BpfProgVerifierInfo::from(attr); + let prog = BpfProg::new(args); + let fd_table = ProcessManager::current_pcb().fd_table(); + let prog = BpfProgVerifier::new(prog, log_info.log_level, &mut []).verify(&fd_table)?; + let file = File::new(Arc::new(prog), FileMode::O_RDWR)?; + let fd = fd_table.write().alloc_fd(file, None).map(|x| x as usize)?; + Ok(fd) +} diff --git a/kernel/src/bpf/prog/util.rs b/kernel/src/bpf/prog/util.rs new file mode 100644 index 000000000..04aa47f3c --- /dev/null +++ b/kernel/src/bpf/prog/util.rs @@ -0,0 +1,112 @@ +use crate::include::bindings::linux_bpf::{bpf_attach_type, bpf_attr, bpf_prog_type}; +use crate::syscall::user_access::{check_and_clone_cstr, UserBufferReader}; +use alloc::string::{String, ToString}; +use alloc::vec::Vec; +use core::ffi::CStr; +use core::fmt::Debug; +use num_traits::FromPrimitive; +use system_error::SystemError; + +bitflags::bitflags! { + + pub struct VerifierLogLevel: u32 { + /// Sets no verifier logging. + const DISABLE = 0; + /// Enables debug verifier logging. + const DEBUG = 1; + /// Enables verbose verifier logging. + const VERBOSE = 2 | Self::DEBUG.bits(); + /// Enables verifier stats. + const STATS = 4; + } +} + +#[derive(Debug)] +pub struct BpfProgVerifierInfo { + /// This attribute specifies the level/detail of the log output. Valid values are. + pub log_level: VerifierLogLevel, + /// This attributes indicates the size of the memory region in bytes + /// indicated by `log_buf` which can safely be written to by the kernel. + pub _log_buf_size: u32, + /// This attributes can be set to a pointer to a memory region + /// allocated/reservedby the loader process where the verifier log will + /// be written to. + /// The detail of the log is set by log_level. The verifier log + /// is often the only indication in addition to the error code of + /// why the syscall command failed to load the program. + /// + /// The log is also written to on success. If the kernel runs out of + /// space in the buffer while loading, the loading process will fail + /// and the command will return with an error code of -ENOSPC. So it + /// is important to correctly size the buffer when enabling logging. + pub _log_buf_ptr: usize, +} + +impl From<&bpf_attr> for BpfProgVerifierInfo { + fn from(attr: &bpf_attr) -> Self { + unsafe { + let u = &attr.__bindgen_anon_3; + Self { + log_level: VerifierLogLevel::from_bits_truncate(u.log_level), + _log_buf_size: u.log_size, + _log_buf_ptr: u.log_buf as usize, + } + } + } +} + +pub struct BpfProgMeta { + pub prog_flags: u32, + pub prog_type: bpf_prog_type, + pub expected_attach_type: bpf_attach_type, + pub insns: Vec, + pub license: String, + pub kern_version: u32, + pub name: String, +} + +impl Debug for BpfProgMeta { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.debug_struct("BpfProgMeta") + .field("prog_flags", &self.prog_flags) + .field("prog_type", &self.prog_type) + .field("expected_attach_type", &self.expected_attach_type) + .field("insns_len", &(self.insns.len() / 8)) + .field("license", &self.license) + .field("kern_version", &self.kern_version) + .field("name", &self.name) + .finish() + } +} + +impl TryFrom<&bpf_attr> for BpfProgMeta { + type Error = SystemError; + fn try_from(attr: &bpf_attr) -> Result { + let u = unsafe { &attr.__bindgen_anon_3 }; + let prog_type = bpf_prog_type::from_u32(u.prog_type).ok_or(SystemError::EINVAL)?; + let expected_attach_type = + bpf_attach_type::from_u32(u.expected_attach_type).ok_or(SystemError::EINVAL)?; + unsafe { + let insns_buf = + UserBufferReader::new(u.insns as *mut u8, u.insn_cnt as usize * 8, true)?; + let insns = insns_buf.read_from_user::(0)?.to_vec(); + let name_slice = + core::slice::from_raw_parts(u.prog_name.as_ptr() as *const u8, u.prog_name.len()); + let prog_name = CStr::from_bytes_until_nul(name_slice) + .map_err(|_| SystemError::EINVAL)? + .to_str() + .map_err(|_| SystemError::EINVAL)? + .to_string(); + let license = check_and_clone_cstr(u.license as *const u8, None)?; + Ok(Self { + prog_flags: u.prog_flags, + prog_type, + expected_attach_type, + insns, + license: license.into_string().map_err(|_| SystemError::EINVAL)?, + kern_version: u.kern_version, + name: prog_name, + }) + } + } +} diff --git a/kernel/src/bpf/prog/verifier.rs b/kernel/src/bpf/prog/verifier.rs new file mode 100644 index 000000000..97cb1542b --- /dev/null +++ b/kernel/src/bpf/prog/verifier.rs @@ -0,0 +1,131 @@ +use super::super::Result; +use crate::bpf::map::BpfMap; +use crate::bpf::prog::util::VerifierLogLevel; +use crate::bpf::prog::BpfProg; +use crate::filesystem::vfs::file::FileDescriptorVec; +use crate::include::bindings::linux_bpf::*; +use crate::libs::casting::DowncastArc; +use crate::libs::rwlock::RwLock; +use alloc::{sync::Arc, vec::Vec}; +use log::{error, info}; +use rbpf::ebpf; +use rbpf::ebpf::to_insn_vec; +use system_error::SystemError; + +/// The BPF program verifier. +/// +/// See https://docs.kernel.org/bpf/verifier.html +#[derive(Debug)] +pub struct BpfProgVerifier<'a> { + prog: BpfProg, + _log_level: VerifierLogLevel, + _log_buf: &'a mut [u8], +} + +impl<'a> BpfProgVerifier<'a> { + pub fn new(prog: BpfProg, log_level: VerifierLogLevel, log_buf: &'a mut [u8]) -> Self { + Self { + prog, + _log_level: log_level, + _log_buf: log_buf, + } + } + /// Relocate the program. + /// + /// This function will relocate the program, and update the program's instructions. + fn relocation(&mut self, fd_table: &Arc>) -> Result<()> { + let instructions = self.prog.insns_mut(); + let mut fmt_insn = to_insn_vec(instructions); + let mut index = 0; + let mut raw_file_ptr = vec![]; + loop { + if index >= fmt_insn.len() { + break; + } + let mut insn = fmt_insn[index].clone(); + if insn.opc == ebpf::LD_DW_IMM { + // relocate the instruction + let mut next_insn = fmt_insn[index + 1].clone(); + // the imm is the map_fd because user lib has already done the relocation + let map_fd = insn.imm as usize; + let src_reg = insn.src; + // See https://www.kernel.org/doc/html/latest/bpf/standardization/instruction-set.html#id23 + let ptr = match src_reg as u32 { + BPF_PSEUDO_MAP_VALUE => { + // dst = map_val(map_by_fd(imm)) + next_imm + // map_val(map) gets the address of the first value in a given map + let file = fd_table + .read() + .get_file_by_fd(map_fd as i32) + .ok_or(SystemError::EBADF)?; + let bpf_map = file + .inode() + .downcast_arc::() + .ok_or(SystemError::EINVAL)?; + let first_value_ptr = + bpf_map.inner_map().lock().first_value_ptr()? as usize; + let offset = next_insn.imm as usize; + info!( + "Relocate for BPF_PSEUDO_MAP_VALUE, instruction index: {}, map_fd: {}", + index, map_fd + ); + Some(first_value_ptr + offset) + } + BPF_PSEUDO_MAP_FD => { + // dst = map_by_fd(imm) + // map_by_fd(imm) means to convert a 32-bit file descriptor into an address of a map + let bpf_map = fd_table + .read() + .get_file_by_fd(map_fd as i32) + .ok_or(SystemError::EBADF)? + .inode() + .downcast_arc::() + .ok_or(SystemError::EINVAL)?; + // todo!(warning: We need release after prog unload) + let map_ptr = Arc::into_raw(bpf_map) as usize; + info!( + "Relocate for BPF_PSEUDO_MAP_FD, instruction index: {}, map_fd: {}, ptr: {:#x}", + index, map_fd, map_ptr + ); + raw_file_ptr.push(map_ptr); + Some(map_ptr) + } + ty => { + error!( + "relocation for ty: {} not implemented, instruction index: {}", + ty, index + ); + None + } + }; + if let Some(ptr) = ptr { + // The current ins store the map_data_ptr low 32 bits, + // the next ins store the map_data_ptr high 32 bits + insn.imm = ptr as i32; + next_insn.imm = (ptr >> 32) as i32; + fmt_insn[index] = insn; + fmt_insn[index + 1] = next_insn; + index += 2; + } else { + index += 1; + } + } else { + index += 1; + } + } + let fmt_insn = fmt_insn + .iter() + .flat_map(|ins| ins.to_vec()) + .collect::>(); + instructions.copy_from_slice(&fmt_insn); + for ptr in raw_file_ptr { + self.prog.insert_map(ptr); + } + Ok(()) + } + + pub fn verify(mut self, fd_table: &Arc>) -> Result { + self.relocation(fd_table)?; + Ok(self.prog) + } +} diff --git a/kernel/src/common/string.h b/kernel/src/common/string.h index 81201624f..cd7032ee1 100644 --- a/kernel/src/common/string.h +++ b/kernel/src/common/string.h @@ -11,3 +11,12 @@ static inline int strlen(const char *s) { } return __res; } + + +static inline int strcmp(const char *s1, const char *s2) { + while (*s1 && *s2 && *s1 == *s2) { + ++s1; + ++s2; + } + return *s1 - *s2; +} \ No newline at end of file diff --git a/kernel/src/debug/kprobe/args.rs b/kernel/src/debug/kprobe/args.rs new file mode 100644 index 000000000..33ca45491 --- /dev/null +++ b/kernel/src/debug/kprobe/args.rs @@ -0,0 +1,66 @@ +use alloc::boxed::Box; +use alloc::string::String; +use kprobe::{CallBackFunc, KprobeBuilder, ProbeArgs}; +use log::warn; +use system_error::SystemError; + +pub struct KprobeInfo { + pub pre_handler: fn(&dyn ProbeArgs), + pub post_handler: fn(&dyn ProbeArgs), + pub fault_handler: Option, + pub event_callback: Option>, + pub symbol: Option, + pub addr: Option, + pub offset: usize, + pub enable: bool, +} + +extern "C" { + fn addr_from_symbol(symbol: *const u8) -> usize; +} + +impl TryFrom for KprobeBuilder { + type Error = SystemError; + fn try_from(kprobe_info: KprobeInfo) -> Result { + // 检查参数: symbol和addr必须有一个但不能同时有 + if kprobe_info.symbol.is_none() && kprobe_info.addr.is_none() { + return Err(SystemError::EINVAL); + } + if kprobe_info.symbol.is_some() && kprobe_info.addr.is_some() { + return Err(SystemError::EINVAL); + } + let func_addr = if let Some(symbol) = kprobe_info.symbol.clone() { + let mut symbol_sting = symbol; + if !symbol_sting.ends_with("\0") { + symbol_sting.push('\0'); + } + let symbol = symbol_sting.as_ptr(); + let func_addr = unsafe { addr_from_symbol(symbol) }; + if func_addr == 0 { + warn!( + "register_kprobe: the symbol: {:?} not found", + kprobe_info.symbol + ); + return Err(SystemError::ENXIO); + } + func_addr + } else { + kprobe_info.addr.unwrap() + }; + let mut builder = KprobeBuilder::new( + kprobe_info.symbol, + func_addr, + kprobe_info.offset, + kprobe_info.pre_handler, + kprobe_info.post_handler, + kprobe_info.enable, + ); + if let Some(fault_handler) = kprobe_info.fault_handler { + builder = builder.with_fault_handler(fault_handler); + } + if let Some(event_callback) = kprobe_info.event_callback { + builder = builder.with_event_callback(event_callback); + } + Ok(builder) + } +} diff --git a/kernel/src/debug/kprobe/mod.rs b/kernel/src/debug/kprobe/mod.rs new file mode 100644 index 000000000..7dc554f88 --- /dev/null +++ b/kernel/src/debug/kprobe/mod.rs @@ -0,0 +1,183 @@ +use crate::debug::kprobe::args::KprobeInfo; +use crate::libs::rwlock::RwLock; +use crate::libs::spinlock::SpinLock; +use alloc::collections::BTreeMap; +use alloc::sync::Arc; +use alloc::vec::Vec; +use kprobe::{Kprobe, KprobeBuilder, KprobeOps, KprobePoint}; +use system_error::SystemError; + +pub mod args; +#[cfg(feature = "kprobe_test")] +mod test; + +pub type LockKprobe = Arc>; +pub static KPROBE_MANAGER: SpinLock = SpinLock::new(KprobeManager::new()); +static KPROBE_POINT_LIST: SpinLock>> = + SpinLock::new(BTreeMap::new()); + +/// 管理所有的kprobe探测点 +#[derive(Debug, Default)] +pub struct KprobeManager { + break_list: BTreeMap>, + debug_list: BTreeMap>, +} + +impl KprobeManager { + pub const fn new() -> Self { + KprobeManager { + break_list: BTreeMap::new(), + debug_list: BTreeMap::new(), + } + } + /// # 插入一个kprobe + /// + /// ## 参数 + /// - `kprobe`: kprobe的实例 + pub fn insert_kprobe(&mut self, kprobe: LockKprobe) { + let probe_point = kprobe.read().probe_point().clone(); + self.insert_break_point(probe_point.break_address(), kprobe.clone()); + self.insert_debug_point(probe_point.debug_address(), kprobe); + } + + /// # 向break_list中插入一个kprobe + /// + /// ## 参数 + /// - `address`: kprobe的地址, 由`KprobePoint::break_address()`或者`KprobeBuilder::probe_addr()`返回 + /// - `kprobe`: kprobe的实例 + fn insert_break_point(&mut self, address: usize, kprobe: LockKprobe) { + let list = self.break_list.entry(address).or_default(); + list.push(kprobe); + } + + /// # 向debug_list中插入一个kprobe + /// + /// ## 参数 + /// - `address`: kprobe的单步执行地址，由`KprobePoint::debug_address()`返回 + /// - `kprobe`: kprobe的实例 + fn insert_debug_point(&mut self, address: usize, kprobe: LockKprobe) { + let list = self.debug_list.entry(address).or_default(); + list.push(kprobe); + } + + pub fn get_break_list(&self, address: usize) -> Option<&Vec> { + self.break_list.get(&address) + } + + pub fn get_debug_list(&self, address: usize) -> Option<&Vec> { + self.debug_list.get(&address) + } + + /// # 返回一个地址上注册的kprobe数量 + /// + /// ## 参数 + /// - `address`: kprobe的地址, 由`KprobePoint::break_address()`或者`KprobeBuilder::probe_addr()`返回 + pub fn kprobe_num(&self, address: usize) -> usize { + self.break_list_len(address) + } + + #[inline] + fn break_list_len(&self, address: usize) -> usize { + self.break_list + .get(&address) + .map(|list| list.len()) + .unwrap_or(0) + } + #[inline] + fn debug_list_len(&self, address: usize) -> usize { + self.debug_list + .get(&address) + .map(|list| list.len()) + .unwrap_or(0) + } + + /// # 移除一个kprobe + /// + /// ## 参数 + /// - `kprobe`: kprobe的实例 + pub fn remove_kprobe(&mut self, kprobe: &LockKprobe) { + let probe_point = kprobe.read().probe_point().clone(); + self.remove_one_break(probe_point.break_address(), kprobe); + self.remove_one_debug(probe_point.debug_address(), kprobe); + } + + /// # 从break_list中移除一个kprobe + /// + /// 如果没有其他kprobe注册在这个地址上，则删除列表 + /// + /// ## 参数 + /// - `address`: kprobe的地址, 由`KprobePoint::break_address()`或者`KprobeBuilder::probe_addr()`返回 + /// - `kprobe`: kprobe的实例 + fn remove_one_break(&mut self, address: usize, kprobe: &LockKprobe) { + if let Some(list) = self.break_list.get_mut(&address) { + list.retain(|x| !Arc::ptr_eq(x, kprobe)); + } + if self.break_list_len(address) == 0 { + self.break_list.remove(&address); + } + } + + /// # 从debug_list中移除一个kprobe + /// + /// 如果没有其他kprobe注册在这个地址上，则删除列表 + /// + /// ## 参数 + /// - `address`: kprobe的单步执行地址，由`KprobePoint::debug_address()`返回 + /// - `kprobe`: kprobe的实例 + fn remove_one_debug(&mut self, address: usize, kprobe: &LockKprobe) { + if let Some(list) = self.debug_list.get_mut(&address) { + list.retain(|x| !Arc::ptr_eq(x, kprobe)); + } + if self.debug_list_len(address) == 0 { + self.debug_list.remove(&address); + } + } +} + +#[cfg(feature = "kprobe_test")] +#[allow(unused)] +/// This function is only used for testing kprobe +pub fn kprobe_test() { + test::kprobe_test(); +} + +/// # 注册一个kprobe +/// +/// 该函数会根据`symbol`查找对应的函数地址，如果找不到则返回错误。 +/// +/// ## 参数 +/// - `kprobe_info`: kprobe的信息 +pub fn register_kprobe(kprobe_info: KprobeInfo) -> Result { + let kprobe_builder = KprobeBuilder::try_from(kprobe_info)?; + let address = kprobe_builder.probe_addr(); + let existed_point = KPROBE_POINT_LIST.lock().get(&address).map(Clone::clone); + let kprobe = match existed_point { + Some(existed_point) => { + kprobe_builder + .with_probe_point(existed_point.clone()) + .install() + .0 + } + None => { + let (kprobe, probe_point) = kprobe_builder.install(); + KPROBE_POINT_LIST.lock().insert(address, probe_point); + kprobe + } + }; + let kprobe = Arc::new(RwLock::new(kprobe)); + KPROBE_MANAGER.lock().insert_kprobe(kprobe.clone()); + Ok(kprobe) +} + +/// # 注销一个kprobe +/// +/// ## 参数 +/// - `kprobe`: 已安装的kprobe +pub fn unregister_kprobe(kprobe: LockKprobe) { + let kprobe_addr = kprobe.read().probe_point().break_address(); + KPROBE_MANAGER.lock().remove_kprobe(&kprobe); + // 如果没有其他kprobe注册在这个地址上，则删除探测点 + if KPROBE_MANAGER.lock().kprobe_num(kprobe_addr) == 0 { + KPROBE_POINT_LIST.lock().remove(&kprobe_addr); + } +} diff --git a/kernel/src/debug/kprobe/test.rs b/kernel/src/debug/kprobe/test.rs new file mode 100644 index 000000000..f23d0e98c --- /dev/null +++ b/kernel/src/debug/kprobe/test.rs @@ -0,0 +1,84 @@ +use crate::arch::interrupt::TrapFrame; +use crate::debug::kprobe::{register_kprobe, unregister_kprobe, KprobeInfo}; +use alloc::string::ToString; +use kprobe::ProbeArgs; +use log::info; + +#[inline(never)] +fn detect_func(x: usize, y: usize) -> usize { + let hart = 0; + info!("detect_func: hart_id: {}, x: {}, y:{}", hart, x, y); + hart +} + +fn pre_handler(regs: &dyn ProbeArgs) { + let pt_regs = regs.as_any().downcast_ref::().unwrap(); + info!( + "call pre_handler, the sp is {:#x}", + pt_regs as *const _ as usize + ); +} + +fn post_handler(regs: &dyn ProbeArgs) { + let pt_regs = regs.as_any().downcast_ref::().unwrap(); + info!( + "call post_handler, the sp is {:#x}", + pt_regs as *const _ as usize + ); +} + +fn fault_handler(regs: &dyn ProbeArgs) { + let pt_regs = regs.as_any().downcast_ref::().unwrap(); + info!( + "call fault_handler, the sp is {:#x}", + pt_regs as *const _ as usize + ); +} + +pub fn kprobe_test() { + info!("kprobe test for [detect_func]: {:#x}", detect_func as usize); + let kprobe_info = KprobeInfo { + pre_handler, + post_handler, + fault_handler: Some(fault_handler), + event_callback: None, + symbol: None, + addr: Some(detect_func as usize), + offset: 0, + enable: true, + }; + let kprobe = register_kprobe(kprobe_info).unwrap(); + + let new_pre_handler = |regs: &dyn ProbeArgs| { + let pt_regs = regs.as_any().downcast_ref::().unwrap(); + info!( + "call new pre_handler, the sp is {:#x}", + pt_regs as *const _ as usize + ); + }; + + let kprobe_info = KprobeInfo { + pre_handler: new_pre_handler, + post_handler, + fault_handler: Some(fault_handler), + event_callback: None, + symbol: Some("dragonos_kernel::debug::kprobe::test::detect_func".to_string()), + addr: None, + offset: 0, + enable: true, + }; + let kprobe2 = register_kprobe(kprobe_info).unwrap(); + info!( + "install 2 kprobes at [detect_func]: {:#x}", + detect_func as usize + ); + detect_func(1, 2); + unregister_kprobe(kprobe); + unregister_kprobe(kprobe2); + info!( + "uninstall 2 kprobes at [detect_func]: {:#x}", + detect_func as usize + ); + detect_func(1, 2); + info!("kprobe test end"); +} diff --git a/kernel/src/debug/mod.rs b/kernel/src/debug/mod.rs index 5516f607f..b24c48bc9 100644 --- a/kernel/src/debug/mod.rs +++ b/kernel/src/debug/mod.rs @@ -1 +1,2 @@ pub mod klog; +pub mod kprobe; diff --git a/kernel/src/debug/traceback/traceback.c b/kernel/src/debug/traceback/traceback.c index d9474eb60..9a2fb7e71 100644 --- a/kernel/src/debug/traceback/traceback.c +++ b/kernel/src/debug/traceback/traceback.c @@ -1,5 +1,6 @@ #include "traceback.h" #include +#include #include int lookup_kallsyms(uint64_t addr, int level) @@ -26,6 +27,18 @@ int lookup_kallsyms(uint64_t addr, int level) return -1; } +uint64_t addr_from_symbol(const char *symbol) +{ + const char *str = (const char *)&kallsyms_names; + for (uint64_t i = 0; i < kallsyms_num; ++i) + { + if (strcmp(&str[kallsyms_names_index[i]], symbol) == 0) + return kallsyms_address[i]; + } + return 0; + +} + /** * @brief 追溯内核栈调用情况 * diff --git a/kernel/src/debug/traceback/traceback.h b/kernel/src/debug/traceback/traceback.h index 837cb502d..d2b6274c3 100644 --- a/kernel/src/debug/traceback/traceback.h +++ b/kernel/src/debug/traceback/traceback.h @@ -14,4 +14,5 @@ extern const char *kallsyms_names __attribute__((weak)); * * @param regs 内核栈结构体 */ -void traceback(struct pt_regs *regs); \ No newline at end of file +void traceback(struct pt_regs *regs); +uint64_t addr_from_symbol(const char *symbol); \ No newline at end of file diff --git a/kernel/src/driver/base/cpu.rs b/kernel/src/driver/base/cpu.rs index 084867306..29e64d7d8 100644 --- a/kernel/src/driver/base/cpu.rs +++ b/kernel/src/driver/base/cpu.rs @@ -11,8 +11,6 @@ use crate::{ libs::rwlock::{RwLock, RwLockReadGuard, RwLockWriteGuard}, }; -use system_error::SystemError; - use super::{ class::Class, device::{ @@ -24,29 +22,36 @@ use super::{ kset::KSet, subsys::SubSysPrivate, }; +use crate::filesystem::sysfs::file::sysfs_emit_str; +use crate::filesystem::sysfs::{Attribute, AttributeGroup, SysFSOpsSupport}; +use crate::filesystem::vfs::syscall::ModeType; +use crate::libs::lazy_init::Lazy; +use system_error::SystemError; -#[inline(always)] -pub fn cpu_device_manager() -> &'static CpuDeviceManager { - return &CpuDeviceManager; -} +static CPU_DEVICE_MANAGER: Lazy = Lazy::new(); #[derive(Debug)] -pub struct CpuDeviceManager; +pub struct CpuDeviceManager { + _root_device: Arc, +} impl CpuDeviceManager { /// 初始化设备驱动模型的CPU子系统 /// /// 参考 https://code.dragonos.org.cn/xref/linux-6.1.9/drivers/base/cpu.c?fi=get_cpu_device#622 - pub fn init(&self) -> Result<(), SystemError> { + pub fn init() -> Result<(), SystemError> { let cpu_subsys = CpuSubSystem::new(); let root_device = CpuSubSystemFakeRootDevice::new(); subsystem_manager() .subsys_system_register( &(cpu_subsys as Arc), - &(root_device as Arc), + &(root_device.clone() as Arc), ) .expect("register cpu subsys failed"); - + let manager = Self { + _root_device: root_device, + }; + CPU_DEVICE_MANAGER.init(manager); return Ok(()); } } @@ -190,6 +195,10 @@ impl Device for CpuSubSystemFakeRootDevice { fn set_dev_parent(&self, dev_parent: Option>) { self.inner.write().device_common.parent = dev_parent; } + + fn attribute_groups(&self) -> Option<&'static [&'static dyn AttributeGroup]> { + Some(&[&AttrGroupCpu]) + } } impl KObject for CpuSubSystemFakeRootDevice { @@ -249,3 +258,70 @@ impl KObject for CpuSubSystemFakeRootDevice { *self.kobj_state_mut() = state; } } + +#[derive(Debug)] +pub struct AttrGroupCpu; + +impl AttributeGroup for AttrGroupCpu { + fn name(&self) -> Option<&str> { + None + } + fn attrs(&self) -> &[&'static dyn Attribute] { + &[&AttrCpuPossible, &AttrCpuOnline] + } + fn is_visible( + &self, + _kobj: Arc, + _attr: &'static dyn Attribute, + ) -> Option { + None + } +} + +#[derive(Debug)] +pub struct AttrCpuPossible; + +impl Attribute for AttrCpuPossible { + fn name(&self) -> &str { + "possible" + } + + fn mode(&self) -> ModeType { + ModeType::S_IRUGO + } + + fn support(&self) -> SysFSOpsSupport { + SysFSOpsSupport::ATTR_SHOW + } + + fn show(&self, _kobj: Arc, buf: &mut [u8]) -> Result { + let cpu_manager = crate::smp::cpu::smp_cpu_manager(); + let cpus = cpu_manager.possible_cpus_count(); + let data = format!("0-{}", cpus - 1); + sysfs_emit_str(buf, &data) + } +} + +#[derive(Debug)] +pub struct AttrCpuOnline; + +impl Attribute for AttrCpuOnline { + fn name(&self) -> &str { + "online" + } + + fn mode(&self) -> ModeType { + ModeType::S_IRUGO + } + + fn support(&self) -> SysFSOpsSupport { + SysFSOpsSupport::ATTR_SHOW + } + + fn show(&self, _kobj: Arc, buf: &mut [u8]) -> Result { + let cpu_manager = crate::smp::cpu::smp_cpu_manager(); + let cpus = cpu_manager.present_cpus_count(); + let data = format!("0-{}", cpus - 1); + sysfs_emit_str(buf, &data) + } +} diff --git a/kernel/src/driver/base/init.rs b/kernel/src/driver/base/init.rs index c28119efc..93a32d817 100644 --- a/kernel/src/driver/base/init.rs +++ b/kernel/src/driver/base/init.rs @@ -3,7 +3,7 @@ use system_error::SystemError; use super::{ class::classes_init, - cpu::cpu_device_manager, + cpu::CpuDeviceManager, device::{bus::buses_init, init::devices_init}, firmware::firmware_init, hypervisor::hypervisor_init, @@ -20,7 +20,7 @@ pub fn driver_init() -> Result<(), SystemError> { hypervisor_init()?; platform_bus_init()?; serio_bus_init()?; - cpu_device_manager().init()?; + CpuDeviceManager::init()?; // 至此，已完成设备驱动模型的初始化 return Ok(()); diff --git a/kernel/src/exception/debug.rs b/kernel/src/exception/debug.rs new file mode 100644 index 000000000..b970d5374 --- /dev/null +++ b/kernel/src/exception/debug.rs @@ -0,0 +1,33 @@ +use crate::arch::interrupt::TrapFrame; +use crate::arch::kprobe::clear_single_step; +use crate::debug::kprobe::KPROBE_MANAGER; +use kprobe::{KprobeOps, ProbeArgs}; +use log::debug; +use system_error::SystemError; + +#[derive(Debug)] +pub struct DebugException; + +impl DebugException { + pub fn handle(frame: &mut TrapFrame) -> Result<(), SystemError> { + Self::post_kprobe_handler(frame) + } + + fn post_kprobe_handler(frame: &mut TrapFrame) -> Result<(), SystemError> { + let pc = frame.debug_address(); + if let Some(kprobe_list) = KPROBE_MANAGER.lock().get_debug_list(pc) { + for kprobe in kprobe_list { + let guard = kprobe.read(); + if guard.is_enabled() { + guard.call_post_handler(frame); + guard.call_event_callback(frame); + } + } + let return_address = kprobe_list[0].read().probe_point().return_address(); + clear_single_step(frame, return_address); + } else { + debug!("There is no kprobe on pc {:#x}", pc); + } + Ok(()) + } +} diff --git a/kernel/src/exception/ebreak.rs b/kernel/src/exception/ebreak.rs new file mode 100644 index 000000000..24adfa137 --- /dev/null +++ b/kernel/src/exception/ebreak.rs @@ -0,0 +1,37 @@ +use crate::arch::interrupt::TrapFrame; +use crate::arch::kprobe::setup_single_step; +use crate::debug::kprobe::KPROBE_MANAGER; +use crate::exception::debug::DebugException; +use kprobe::{KprobeOps, ProbeArgs}; +use system_error::SystemError; + +#[derive(Debug)] +pub struct EBreak; + +impl EBreak { + pub fn handle(frame: &mut TrapFrame) -> Result<(), SystemError> { + Self::kprobe_handler(frame) + } + fn kprobe_handler(frame: &mut TrapFrame) -> Result<(), SystemError> { + let break_addr = frame.break_address(); + let guard = KPROBE_MANAGER.lock(); + let kprobe_list = guard.get_break_list(break_addr); + if let Some(kprobe_list) = kprobe_list { + for kprobe in kprobe_list { + let guard = kprobe.read(); + if guard.is_enabled() { + guard.call_pre_handler(frame); + } + } + let single_step_address = kprobe_list[0].read().probe_point().single_step_address(); + // setup_single_step + setup_single_step(frame, single_step_address); + } else { + // For some architectures, they do not support single step execution, + // and we need to use breakpoint exceptions to simulate + drop(guard); + DebugException::handle(frame)?; + } + Ok(()) + } +} diff --git a/kernel/src/exception/mod.rs b/kernel/src/exception/mod.rs index c7df823e3..8eb14dd6b 100644 --- a/kernel/src/exception/mod.rs +++ b/kernel/src/exception/mod.rs @@ -4,7 +4,9 @@ use system_error::SystemError; use crate::arch::CurrentIrqArch; +pub mod debug; pub mod dummychip; +pub mod ebreak; pub mod handle; pub mod init; pub mod ipi; diff --git a/kernel/src/filesystem/vfs/file.rs b/kernel/src/filesystem/vfs/file.rs index d15167bad..4b3d75b6f 100644 --- a/kernel/src/filesystem/vfs/file.rs +++ b/kernel/src/filesystem/vfs/file.rs @@ -11,6 +11,8 @@ use system_error::SystemError; use super::{Dirent, FileType, IndexNode, InodeId, Metadata, SpecialNodeData}; use crate::filesystem::eventfd::EventFdInode; +use crate::libs::lazy_init::Lazy; +use crate::perf::PerfEventInode; use crate::{ arch::MMArch, driver::{ @@ -125,7 +127,7 @@ impl FileMode { /// 页面缓存 pub struct PageCache { xarray: SpinLock>>, - inode: Option>, + inode: Lazy>, } impl core::fmt::Debug for PageCache { @@ -148,13 +150,19 @@ impl PageCache { pub fn new(inode: Option>) -> Arc { let page_cache = Self { xarray: SpinLock::new(XArray::new()), - inode, + inode: { + let v: Lazy> = Lazy::new(); + if let Some(inode) = inode { + v.init(inode); + } + v + }, }; Arc::new(page_cache) } pub fn inode(&self) -> Option> { - self.inode.clone() + self.inode.try_get().cloned() } pub fn add_page(&self, offset: usize, page: &Arc) { @@ -176,8 +184,12 @@ impl PageCache { cursor.remove(); } - pub fn set_inode(&mut self, inode: Weak) { - self.inode = Some(inode) + pub fn set_inode(&self, inode: Weak) -> Result<(), SystemError> { + if self.inode.initialized() { + return Err(SystemError::EINVAL); + } + self.inode.init(inode); + Ok(()) } } @@ -603,11 +615,15 @@ impl File { inode.inner().lock().remove_epoll(epoll) } _ => { + let inode = self.inode.downcast_ref::(); + if let Some(inode) = inode { + return inode.remove_epoll(epoll); + } let inode = self .inode - .downcast_ref::() + .downcast_ref::() .ok_or(SystemError::ENOSYS)?; - inode.remove_epoll(epoll) + return inode.remove_epoll(epoll); } } } @@ -745,7 +761,6 @@ impl FileDescriptorVec { // 把文件描述符数组对应位置设置为空 let file = self.fds[fd as usize].take().unwrap(); - return Ok(file); } diff --git a/kernel/src/filesystem/vfs/mod.rs b/kernel/src/filesystem/vfs/mod.rs index 00ce6ba50..51d2c4220 100644 --- a/kernel/src/filesystem/vfs/mod.rs +++ b/kernel/src/filesystem/vfs/mod.rs @@ -125,6 +125,9 @@ bitflags! { } pub trait IndexNode: Any + Sync + Send + Debug + CastFromSync { + fn mmap(&self, _start: usize, _len: usize, _offset: usize) -> Result<(), SystemError> { + return Err(SystemError::ENOSYS); + } /// @brief 打开文件 /// /// @return 成功：Ok() diff --git a/kernel/src/include/bindings/linux_bpf.rs b/kernel/src/include/bindings/linux_bpf.rs new file mode 100644 index 000000000..e3b2b79da --- /dev/null +++ b/kernel/src/include/bindings/linux_bpf.rs @@ -0,0 +1,2430 @@ +/* automatically generated by rust-bindgen 0.69.4 */ + +#[repr(C)] +#[derive(Copy, Clone, Debug, Default, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub struct __BindgenBitfieldUnit { + storage: Storage, +} +impl __BindgenBitfieldUnit { + #[inline] + pub const fn new(storage: Storage) -> Self { + Self { storage } + } +} +impl __BindgenBitfieldUnit +where + Storage: AsRef<[u8]> + AsMut<[u8]>, +{ + #[inline] + pub fn get_bit(&self, index: usize) -> bool { + debug_assert!(index / 8 < self.storage.as_ref().len()); + let byte_index = index / 8; + let byte = self.storage.as_ref()[byte_index]; + let bit_index = if cfg!(target_endian = "big") { + 7 - (index % 8) + } else { + index % 8 + }; + let mask = 1 << bit_index; + byte & mask == mask + } + #[inline] + pub fn set_bit(&mut self, index: usize, val: bool) { + debug_assert!(index / 8 < self.storage.as_ref().len()); + let byte_index = index / 8; + let byte = &mut self.storage.as_mut()[byte_index]; + let bit_index = if cfg!(target_endian = "big") { + 7 - (index % 8) + } else { + index % 8 + }; + let mask = 1 << bit_index; + if val { + *byte |= mask; + } else { + *byte &= !mask; + } + } + #[inline] + pub fn get(&self, bit_offset: usize, bit_width: u8) -> u64 { + debug_assert!(bit_width <= 64); + debug_assert!(bit_offset / 8 < self.storage.as_ref().len()); + debug_assert!((bit_offset + (bit_width as usize)) / 8 <= self.storage.as_ref().len()); + let mut val = 0; + for i in 0..(bit_width as usize) { + if self.get_bit(i + bit_offset) { + let index = if cfg!(target_endian = "big") { + bit_width as usize - 1 - i + } else { + i + }; + val |= 1 << index; + } + } + val + } + #[inline] + pub fn set(&mut self, bit_offset: usize, bit_width: u8, val: u64) { + debug_assert!(bit_width <= 64); + debug_assert!(bit_offset / 8 < self.storage.as_ref().len()); + debug_assert!((bit_offset + (bit_width as usize)) / 8 <= self.storage.as_ref().len()); + for i in 0..(bit_width as usize) { + let mask = 1 << i; + let val_bit_is_set = val & mask == mask; + let index = if cfg!(target_endian = "big") { + bit_width as usize - 1 - i + } else { + i + }; + self.set_bit(index + bit_offset, val_bit_is_set); + } + } +} +#[repr(C)] +#[derive(Default)] +pub struct __IncompleteArrayField(::core::marker::PhantomData, [T; 0]); +impl __IncompleteArrayField { + #[inline] + pub const fn new() -> Self { + __IncompleteArrayField(::core::marker::PhantomData, []) + } + #[inline] + pub fn as_ptr(&self) -> *const T { + self as *const _ as *const T + } + #[inline] + pub fn as_mut_ptr(&mut self) -> *mut T { + self as *mut _ as *mut T + } + #[inline] + pub unsafe fn as_slice(&self, len: usize) -> &[T] { + ::core::slice::from_raw_parts(self.as_ptr(), len) + } + #[inline] + pub unsafe fn as_mut_slice(&mut self, len: usize) -> &mut [T] { + ::core::slice::from_raw_parts_mut(self.as_mut_ptr(), len) + } +} +impl ::core::fmt::Debug for __IncompleteArrayField { + fn fmt(&self, fmt: &mut ::core::fmt::Formatter<'_>) -> ::core::fmt::Result { + fmt.write_str("__IncompleteArrayField") + } +} +pub const SO_ATTACH_BPF: u32 = 50; +pub const SO_DETACH_BPF: u32 = 27; +pub const BPF_LD: u32 = 0; +pub const BPF_LDX: u32 = 1; +pub const BPF_ST: u32 = 2; +pub const BPF_STX: u32 = 3; +pub const BPF_ALU: u32 = 4; +pub const BPF_JMP: u32 = 5; +pub const BPF_W: u32 = 0; +pub const BPF_H: u32 = 8; +pub const BPF_B: u32 = 16; +pub const BPF_K: u32 = 0; +pub const BPF_ALU64: u32 = 7; +pub const BPF_DW: u32 = 24; +pub const BPF_CALL: u32 = 128; +pub const BPF_F_ALLOW_OVERRIDE: u32 = 1; +pub const BPF_F_ALLOW_MULTI: u32 = 2; +pub const BPF_F_REPLACE: u32 = 4; +pub const BPF_F_BEFORE: u32 = 8; +pub const BPF_F_AFTER: u32 = 16; +pub const BPF_F_ID: u32 = 32; +pub const BPF_F_STRICT_ALIGNMENT: u32 = 1; +pub const BPF_F_ANY_ALIGNMENT: u32 = 2; +pub const BPF_F_TEST_RND_HI32: u32 = 4; +pub const BPF_F_TEST_STATE_FREQ: u32 = 8; +pub const BPF_F_SLEEPABLE: u32 = 16; +pub const BPF_F_XDP_HAS_FRAGS: u32 = 32; +pub const BPF_F_XDP_DEV_BOUND_ONLY: u32 = 64; +pub const BPF_F_TEST_REG_INVARIANTS: u32 = 128; +pub const BPF_F_NETFILTER_IP_DEFRAG: u32 = 1; +pub const BPF_PSEUDO_MAP_FD: u32 = 1; +pub const BPF_PSEUDO_MAP_IDX: u32 = 5; +pub const BPF_PSEUDO_MAP_VALUE: u32 = 2; +pub const BPF_PSEUDO_MAP_IDX_VALUE: u32 = 6; +pub const BPF_PSEUDO_BTF_ID: u32 = 3; +pub const BPF_PSEUDO_FUNC: u32 = 4; +pub const BPF_PSEUDO_CALL: u32 = 1; +pub const BPF_PSEUDO_KFUNC_CALL: u32 = 2; +pub const BPF_F_QUERY_EFFECTIVE: u32 = 1; +pub const BPF_F_TEST_RUN_ON_CPU: u32 = 1; +pub const BPF_F_TEST_XDP_LIVE_FRAMES: u32 = 2; +pub const BTF_INT_SIGNED: u32 = 1; +pub const BTF_INT_CHAR: u32 = 2; +pub const BTF_INT_BOOL: u32 = 4; +pub const NLMSG_ALIGNTO: u32 = 4; +pub const XDP_FLAGS_UPDATE_IF_NOEXIST: u32 = 1; +pub const XDP_FLAGS_SKB_MODE: u32 = 2; +pub const XDP_FLAGS_DRV_MODE: u32 = 4; +pub const XDP_FLAGS_HW_MODE: u32 = 8; +pub const XDP_FLAGS_REPLACE: u32 = 16; +pub const XDP_FLAGS_MODES: u32 = 14; +pub const XDP_FLAGS_MASK: u32 = 31; +pub const PERF_MAX_STACK_DEPTH: u32 = 127; +pub const PERF_MAX_CONTEXTS_PER_STACK: u32 = 8; +pub const PERF_FLAG_FD_NO_GROUP: u32 = 1; +pub const PERF_FLAG_FD_OUTPUT: u32 = 2; +pub const PERF_FLAG_PID_CGROUP: u32 = 4; +pub const PERF_FLAG_FD_CLOEXEC: u32 = 8; +pub const TC_H_MAJ_MASK: u32 = 4294901760; +pub const TC_H_MIN_MASK: u32 = 65535; +pub const TC_H_UNSPEC: u32 = 0; +pub const TC_H_ROOT: u32 = 4294967295; +pub const TC_H_INGRESS: u32 = 4294967281; +pub const TC_H_CLSACT: u32 = 4294967281; +pub const TC_H_MIN_PRIORITY: u32 = 65504; +pub const TC_H_MIN_INGRESS: u32 = 65522; +pub const TC_H_MIN_EGRESS: u32 = 65523; +pub const TCA_BPF_FLAG_ACT_DIRECT: u32 = 1; +pub type __u8 = ::core::ffi::c_uchar; +pub type __s16 = ::core::ffi::c_short; +pub type __u16 = ::core::ffi::c_ushort; +pub type __s32 = ::core::ffi::c_int; +pub type __u32 = ::core::ffi::c_uint; +pub type __s64 = ::core::ffi::c_longlong; +pub type __u64 = ::core::ffi::c_ulonglong; +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct bpf_insn { + pub code: __u8, + pub _bitfield_align_1: [u8; 0], + pub _bitfield_1: __BindgenBitfieldUnit<[u8; 1usize]>, + pub off: __s16, + pub imm: __s32, +} +impl bpf_insn { + #[inline] + pub fn dst_reg(&self) -> __u8 { + unsafe { ::core::mem::transmute(self._bitfield_1.get(0usize, 4u8) as u8) } + } + #[inline] + pub fn set_dst_reg(&mut self, val: __u8) { + unsafe { + let val: u8 = ::core::mem::transmute(val); + self._bitfield_1.set(0usize, 4u8, val as u64) + } + } + #[inline] + pub fn src_reg(&self) -> __u8 { + unsafe { ::core::mem::transmute(self._bitfield_1.get(4usize, 4u8) as u8) } + } + #[inline] + pub fn set_src_reg(&mut self, val: __u8) { + unsafe { + let val: u8 = ::core::mem::transmute(val); + self._bitfield_1.set(4usize, 4u8, val as u64) + } + } + #[inline] + pub fn new_bitfield_1(dst_reg: __u8, src_reg: __u8) -> __BindgenBitfieldUnit<[u8; 1usize]> { + let mut __bindgen_bitfield_unit: __BindgenBitfieldUnit<[u8; 1usize]> = Default::default(); + __bindgen_bitfield_unit.set(0usize, 4u8, { + let dst_reg: u8 = unsafe { ::core::mem::transmute(dst_reg) }; + dst_reg as u64 + }); + __bindgen_bitfield_unit.set(4usize, 4u8, { + let src_reg: u8 = unsafe { ::core::mem::transmute(src_reg) }; + src_reg as u64 + }); + __bindgen_bitfield_unit + } +} +#[repr(C)] +#[derive(Debug)] +pub struct bpf_lpm_trie_key { + pub prefixlen: __u32, + pub data: __IncompleteArrayField<__u8>, +} +impl bpf_cmd { + pub const BPF_PROG_RUN: bpf_cmd = bpf_cmd::BPF_PROG_TEST_RUN; +} +#[repr(u32)] +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq, FromPrimitive)] +pub enum bpf_cmd { + BPF_MAP_CREATE = 0, + BPF_MAP_LOOKUP_ELEM = 1, + BPF_MAP_UPDATE_ELEM = 2, + BPF_MAP_DELETE_ELEM = 3, + BPF_MAP_GET_NEXT_KEY = 4, + BPF_PROG_LOAD = 5, + BPF_OBJ_PIN = 6, + BPF_OBJ_GET = 7, + BPF_PROG_ATTACH = 8, + BPF_PROG_DETACH = 9, + BPF_PROG_TEST_RUN = 10, + BPF_PROG_GET_NEXT_ID = 11, + BPF_MAP_GET_NEXT_ID = 12, + BPF_PROG_GET_FD_BY_ID = 13, + BPF_MAP_GET_FD_BY_ID = 14, + BPF_OBJ_GET_INFO_BY_FD = 15, + BPF_PROG_QUERY = 16, + BPF_RAW_TRACEPOINT_OPEN = 17, + BPF_BTF_LOAD = 18, + BPF_BTF_GET_FD_BY_ID = 19, + BPF_TASK_FD_QUERY = 20, + BPF_MAP_LOOKUP_AND_DELETE_ELEM = 21, + BPF_MAP_FREEZE = 22, + BPF_BTF_GET_NEXT_ID = 23, + BPF_MAP_LOOKUP_BATCH = 24, + BPF_MAP_LOOKUP_AND_DELETE_BATCH = 25, + BPF_MAP_UPDATE_BATCH = 26, + BPF_MAP_DELETE_BATCH = 27, + BPF_LINK_CREATE = 28, + BPF_LINK_UPDATE = 29, + BPF_LINK_GET_FD_BY_ID = 30, + BPF_LINK_GET_NEXT_ID = 31, + BPF_ENABLE_STATS = 32, + BPF_ITER_CREATE = 33, + BPF_LINK_DETACH = 34, + BPF_PROG_BIND_MAP = 35, + BPF_TOKEN_CREATE = 36, + __MAX_BPF_CMD = 37, +} +impl bpf_map_type { + pub const BPF_MAP_TYPE_CGROUP_STORAGE: bpf_map_type = + bpf_map_type::BPF_MAP_TYPE_CGROUP_STORAGE_DEPRECATED; +} +impl bpf_map_type { + pub const BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE: bpf_map_type = + bpf_map_type::BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE_DEPRECATED; +} +#[repr(u32)] +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq, FromPrimitive)] +pub enum bpf_map_type { + BPF_MAP_TYPE_UNSPEC = 0, + BPF_MAP_TYPE_HASH = 1, + BPF_MAP_TYPE_ARRAY = 2, + BPF_MAP_TYPE_PROG_ARRAY = 3, + BPF_MAP_TYPE_PERF_EVENT_ARRAY = 4, + BPF_MAP_TYPE_PERCPU_HASH = 5, + BPF_MAP_TYPE_PERCPU_ARRAY = 6, + BPF_MAP_TYPE_STACK_TRACE = 7, + BPF_MAP_TYPE_CGROUP_ARRAY = 8, + BPF_MAP_TYPE_LRU_HASH = 9, + BPF_MAP_TYPE_LRU_PERCPU_HASH = 10, + BPF_MAP_TYPE_LPM_TRIE = 11, + BPF_MAP_TYPE_ARRAY_OF_MAPS = 12, + BPF_MAP_TYPE_HASH_OF_MAPS = 13, + BPF_MAP_TYPE_DEVMAP = 14, + BPF_MAP_TYPE_SOCKMAP = 15, + BPF_MAP_TYPE_CPUMAP = 16, + BPF_MAP_TYPE_XSKMAP = 17, + BPF_MAP_TYPE_SOCKHASH = 18, + BPF_MAP_TYPE_CGROUP_STORAGE_DEPRECATED = 19, + BPF_MAP_TYPE_REUSEPORT_SOCKARRAY = 20, + BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE_DEPRECATED = 21, + BPF_MAP_TYPE_QUEUE = 22, + BPF_MAP_TYPE_STACK = 23, + BPF_MAP_TYPE_SK_STORAGE = 24, + BPF_MAP_TYPE_DEVMAP_HASH = 25, + BPF_MAP_TYPE_STRUCT_OPS = 26, + BPF_MAP_TYPE_RINGBUF = 27, + BPF_MAP_TYPE_INODE_STORAGE = 28, + BPF_MAP_TYPE_TASK_STORAGE = 29, + BPF_MAP_TYPE_BLOOM_FILTER = 30, + BPF_MAP_TYPE_USER_RINGBUF = 31, + BPF_MAP_TYPE_CGRP_STORAGE = 32, + BPF_MAP_TYPE_ARENA = 33, + __MAX_BPF_MAP_TYPE = 34, +} +#[repr(u32)] +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq, FromPrimitive)] +pub enum bpf_prog_type { + BPF_PROG_TYPE_UNSPEC = 0, + BPF_PROG_TYPE_SOCKET_FILTER = 1, + BPF_PROG_TYPE_KPROBE = 2, + BPF_PROG_TYPE_SCHED_CLS = 3, + BPF_PROG_TYPE_SCHED_ACT = 4, + BPF_PROG_TYPE_TRACEPOINT = 5, + BPF_PROG_TYPE_XDP = 6, + BPF_PROG_TYPE_PERF_EVENT = 7, + BPF_PROG_TYPE_CGROUP_SKB = 8, + BPF_PROG_TYPE_CGROUP_SOCK = 9, + BPF_PROG_TYPE_LWT_IN = 10, + BPF_PROG_TYPE_LWT_OUT = 11, + BPF_PROG_TYPE_LWT_XMIT = 12, + BPF_PROG_TYPE_SOCK_OPS = 13, + BPF_PROG_TYPE_SK_SKB = 14, + BPF_PROG_TYPE_CGROUP_DEVICE = 15, + BPF_PROG_TYPE_SK_MSG = 16, + BPF_PROG_TYPE_RAW_TRACEPOINT = 17, + BPF_PROG_TYPE_CGROUP_SOCK_ADDR = 18, + BPF_PROG_TYPE_LWT_SEG6LOCAL = 19, + BPF_PROG_TYPE_LIRC_MODE2 = 20, + BPF_PROG_TYPE_SK_REUSEPORT = 21, + BPF_PROG_TYPE_FLOW_DISSECTOR = 22, + BPF_PROG_TYPE_CGROUP_SYSCTL = 23, + BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE = 24, + BPF_PROG_TYPE_CGROUP_SOCKOPT = 25, + BPF_PROG_TYPE_TRACING = 26, + BPF_PROG_TYPE_STRUCT_OPS = 27, + BPF_PROG_TYPE_EXT = 28, + BPF_PROG_TYPE_LSM = 29, + BPF_PROG_TYPE_SK_LOOKUP = 30, + BPF_PROG_TYPE_SYSCALL = 31, + BPF_PROG_TYPE_NETFILTER = 32, + __MAX_BPF_PROG_TYPE = 33, +} +#[repr(u32)] +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq, FromPrimitive)] +pub enum bpf_attach_type { + BPF_CGROUP_INET_INGRESS = 0, + BPF_CGROUP_INET_EGRESS = 1, + BPF_CGROUP_INET_SOCK_CREATE = 2, + BPF_CGROUP_SOCK_OPS = 3, + BPF_SK_SKB_STREAM_PARSER = 4, + BPF_SK_SKB_STREAM_VERDICT = 5, + BPF_CGROUP_DEVICE = 6, + BPF_SK_MSG_VERDICT = 7, + BPF_CGROUP_INET4_BIND = 8, + BPF_CGROUP_INET6_BIND = 9, + BPF_CGROUP_INET4_CONNECT = 10, + BPF_CGROUP_INET6_CONNECT = 11, + BPF_CGROUP_INET4_POST_BIND = 12, + BPF_CGROUP_INET6_POST_BIND = 13, + BPF_CGROUP_UDP4_SENDMSG = 14, + BPF_CGROUP_UDP6_SENDMSG = 15, + BPF_LIRC_MODE2 = 16, + BPF_FLOW_DISSECTOR = 17, + BPF_CGROUP_SYSCTL = 18, + BPF_CGROUP_UDP4_RECVMSG = 19, + BPF_CGROUP_UDP6_RECVMSG = 20, + BPF_CGROUP_GETSOCKOPT = 21, + BPF_CGROUP_SETSOCKOPT = 22, + BPF_TRACE_RAW_TP = 23, + BPF_TRACE_FENTRY = 24, + BPF_TRACE_FEXIT = 25, + BPF_MODIFY_RETURN = 26, + BPF_LSM_MAC = 27, + BPF_TRACE_ITER = 28, + BPF_CGROUP_INET4_GETPEERNAME = 29, + BPF_CGROUP_INET6_GETPEERNAME = 30, + BPF_CGROUP_INET4_GETSOCKNAME = 31, + BPF_CGROUP_INET6_GETSOCKNAME = 32, + BPF_XDP_DEVMAP = 33, + BPF_CGROUP_INET_SOCK_RELEASE = 34, + BPF_XDP_CPUMAP = 35, + BPF_SK_LOOKUP = 36, + BPF_XDP = 37, + BPF_SK_SKB_VERDICT = 38, + BPF_SK_REUSEPORT_SELECT = 39, + BPF_SK_REUSEPORT_SELECT_OR_MIGRATE = 40, + BPF_PERF_EVENT = 41, + BPF_TRACE_KPROBE_MULTI = 42, + BPF_LSM_CGROUP = 43, + BPF_STRUCT_OPS = 44, + BPF_NETFILTER = 45, + BPF_TCX_INGRESS = 46, + BPF_TCX_EGRESS = 47, + BPF_TRACE_UPROBE_MULTI = 48, + BPF_CGROUP_UNIX_CONNECT = 49, + BPF_CGROUP_UNIX_SENDMSG = 50, + BPF_CGROUP_UNIX_RECVMSG = 51, + BPF_CGROUP_UNIX_GETPEERNAME = 52, + BPF_CGROUP_UNIX_GETSOCKNAME = 53, + BPF_NETKIT_PRIMARY = 54, + BPF_NETKIT_PEER = 55, + __MAX_BPF_ATTACH_TYPE = 56, +} +#[repr(u32)] +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] +pub enum bpf_link_type { + BPF_LINK_TYPE_UNSPEC = 0, + BPF_LINK_TYPE_RAW_TRACEPOINT = 1, + BPF_LINK_TYPE_TRACING = 2, + BPF_LINK_TYPE_CGROUP = 3, + BPF_LINK_TYPE_ITER = 4, + BPF_LINK_TYPE_NETNS = 5, + BPF_LINK_TYPE_XDP = 6, + BPF_LINK_TYPE_PERF_EVENT = 7, + BPF_LINK_TYPE_KPROBE_MULTI = 8, + BPF_LINK_TYPE_STRUCT_OPS = 9, + BPF_LINK_TYPE_NETFILTER = 10, + BPF_LINK_TYPE_TCX = 11, + BPF_LINK_TYPE_UPROBE_MULTI = 12, + BPF_LINK_TYPE_NETKIT = 13, + __MAX_BPF_LINK_TYPE = 14, +} +pub const BPF_F_KPROBE_MULTI_RETURN: _bindgen_ty_2 = 1; +pub type _bindgen_ty_2 = ::core::ffi::c_uint; +pub const BPF_F_UPROBE_MULTI_RETURN: _bindgen_ty_3 = 1; +pub type _bindgen_ty_3 = ::core::ffi::c_uint; +pub const BPF_ANY: _bindgen_ty_4 = 0; +pub const BPF_NOEXIST: _bindgen_ty_4 = 1; +pub const BPF_EXIST: _bindgen_ty_4 = 2; +pub const BPF_F_LOCK: _bindgen_ty_4 = 4; +pub type _bindgen_ty_4 = ::core::ffi::c_uint; +pub const BPF_F_NO_PREALLOC: _bindgen_ty_5 = 1; +pub const BPF_F_NO_COMMON_LRU: _bindgen_ty_5 = 2; +pub const BPF_F_NUMA_NODE: _bindgen_ty_5 = 4; +pub const BPF_F_RDONLY: _bindgen_ty_5 = 8; +pub const BPF_F_WRONLY: _bindgen_ty_5 = 16; +pub const BPF_F_STACK_BUILD_ID: _bindgen_ty_5 = 32; +pub const BPF_F_ZERO_SEED: _bindgen_ty_5 = 64; +pub const BPF_F_RDONLY_PROG: _bindgen_ty_5 = 128; +pub const BPF_F_WRONLY_PROG: _bindgen_ty_5 = 256; +pub const BPF_F_CLONE: _bindgen_ty_5 = 512; +pub const BPF_F_MMAPABLE: _bindgen_ty_5 = 1024; +pub const BPF_F_PRESERVE_ELEMS: _bindgen_ty_5 = 2048; +pub const BPF_F_INNER_MAP: _bindgen_ty_5 = 4096; +pub const BPF_F_LINK: _bindgen_ty_5 = 8192; +pub const BPF_F_PATH_FD: _bindgen_ty_5 = 16384; +pub const BPF_F_VTYPE_BTF_OBJ_FD: _bindgen_ty_5 = 32768; +pub const BPF_F_TOKEN_FD: _bindgen_ty_5 = 65536; +pub const BPF_F_SEGV_ON_FAULT: _bindgen_ty_5 = 131072; +pub const BPF_F_NO_USER_CONV: _bindgen_ty_5 = 262144; +pub type _bindgen_ty_5 = ::core::ffi::c_uint; +#[repr(u32)] +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] +pub enum bpf_stats_type { + BPF_STATS_RUN_TIME = 0, +} +#[repr(C)] +#[derive(Copy, Clone)] +pub union bpf_attr { + pub __bindgen_anon_1: bpf_attr__bindgen_ty_1, + pub __bindgen_anon_2: bpf_attr__bindgen_ty_2, + pub batch: bpf_attr__bindgen_ty_3, + pub __bindgen_anon_3: bpf_attr__bindgen_ty_4, + pub __bindgen_anon_4: bpf_attr__bindgen_ty_5, + pub __bindgen_anon_5: bpf_attr__bindgen_ty_6, + pub test: bpf_attr__bindgen_ty_7, + pub __bindgen_anon_6: bpf_attr__bindgen_ty_8, + pub info: bpf_attr__bindgen_ty_9, + pub query: bpf_attr__bindgen_ty_10, + pub raw_tracepoint: bpf_attr__bindgen_ty_11, + pub __bindgen_anon_7: bpf_attr__bindgen_ty_12, + pub task_fd_query: bpf_attr__bindgen_ty_13, + pub link_create: bpf_attr__bindgen_ty_14, + pub link_update: bpf_attr__bindgen_ty_15, + pub link_detach: bpf_attr__bindgen_ty_16, + pub enable_stats: bpf_attr__bindgen_ty_17, + pub iter_create: bpf_attr__bindgen_ty_18, + pub prog_bind_map: bpf_attr__bindgen_ty_19, + pub token_create: bpf_attr__bindgen_ty_20, +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct bpf_attr__bindgen_ty_1 { + pub map_type: __u32, + pub key_size: __u32, + pub value_size: __u32, + pub max_entries: __u32, + pub map_flags: __u32, + pub inner_map_fd: __u32, + pub numa_node: __u32, + pub map_name: [::core::ffi::c_char; 16usize], + pub map_ifindex: __u32, + pub btf_fd: __u32, + pub btf_key_type_id: __u32, + pub btf_value_type_id: __u32, + pub btf_vmlinux_value_type_id: __u32, + pub map_extra: __u64, + pub value_type_btf_obj_fd: __s32, + pub map_token_fd: __s32, +} +#[repr(C)] +#[derive(Copy, Clone)] +pub struct bpf_attr__bindgen_ty_2 { + pub map_fd: __u32, + pub key: __u64, + pub __bindgen_anon_1: bpf_attr__bindgen_ty_2__bindgen_ty_1, + pub flags: __u64, +} +#[repr(C)] +#[derive(Copy, Clone)] +pub union bpf_attr__bindgen_ty_2__bindgen_ty_1 { + pub value: __u64, + pub next_key: __u64, +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct bpf_attr__bindgen_ty_3 { + pub in_batch: __u64, + pub out_batch: __u64, + pub keys: __u64, + pub values: __u64, + pub count: __u32, + pub map_fd: __u32, + pub elem_flags: __u64, + pub flags: __u64, +} +#[repr(C)] +#[derive(Copy, Clone)] +pub struct bpf_attr__bindgen_ty_4 { + pub prog_type: __u32, + pub insn_cnt: __u32, + pub insns: __u64, + pub license: __u64, + pub log_level: __u32, + pub log_size: __u32, + pub log_buf: __u64, + pub kern_version: __u32, + pub prog_flags: __u32, + pub prog_name: [::core::ffi::c_char; 16usize], + pub prog_ifindex: __u32, + pub expected_attach_type: __u32, + pub prog_btf_fd: __u32, + pub func_info_rec_size: __u32, + pub func_info: __u64, + pub func_info_cnt: __u32, + pub line_info_rec_size: __u32, + pub line_info: __u64, + pub line_info_cnt: __u32, + pub attach_btf_id: __u32, + pub __bindgen_anon_1: bpf_attr__bindgen_ty_4__bindgen_ty_1, + pub core_relo_cnt: __u32, + pub fd_array: __u64, + pub core_relos: __u64, + pub core_relo_rec_size: __u32, + pub log_true_size: __u32, + pub prog_token_fd: __s32, +} +#[repr(C)] +#[derive(Copy, Clone)] +pub union bpf_attr__bindgen_ty_4__bindgen_ty_1 { + pub attach_prog_fd: __u32, + pub attach_btf_obj_fd: __u32, +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct bpf_attr__bindgen_ty_5 { + pub pathname: __u64, + pub bpf_fd: __u32, + pub file_flags: __u32, + pub path_fd: __s32, +} +#[repr(C)] +#[derive(Copy, Clone)] +pub struct bpf_attr__bindgen_ty_6 { + pub __bindgen_anon_1: bpf_attr__bindgen_ty_6__bindgen_ty_1, + pub attach_bpf_fd: __u32, + pub attach_type: __u32, + pub attach_flags: __u32, + pub replace_bpf_fd: __u32, + pub __bindgen_anon_2: bpf_attr__bindgen_ty_6__bindgen_ty_2, + pub expected_revision: __u64, +} +#[repr(C)] +#[derive(Copy, Clone)] +pub union bpf_attr__bindgen_ty_6__bindgen_ty_1 { + pub target_fd: __u32, + pub target_ifindex: __u32, +} +#[repr(C)] +#[derive(Copy, Clone)] +pub union bpf_attr__bindgen_ty_6__bindgen_ty_2 { + pub relative_fd: __u32, + pub relative_id: __u32, +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct bpf_attr__bindgen_ty_7 { + pub prog_fd: __u32, + pub retval: __u32, + pub data_size_in: __u32, + pub data_size_out: __u32, + pub data_in: __u64, + pub data_out: __u64, + pub repeat: __u32, + pub duration: __u32, + pub ctx_size_in: __u32, + pub ctx_size_out: __u32, + pub ctx_in: __u64, + pub ctx_out: __u64, + pub flags: __u32, + pub cpu: __u32, + pub batch_size: __u32, +} +#[repr(C)] +#[derive(Copy, Clone)] +pub struct bpf_attr__bindgen_ty_8 { + pub __bindgen_anon_1: bpf_attr__bindgen_ty_8__bindgen_ty_1, + pub next_id: __u32, + pub open_flags: __u32, +} +#[repr(C)] +#[derive(Copy, Clone)] +pub union bpf_attr__bindgen_ty_8__bindgen_ty_1 { + pub start_id: __u32, + pub prog_id: __u32, + pub map_id: __u32, + pub btf_id: __u32, + pub link_id: __u32, +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct bpf_attr__bindgen_ty_9 { + pub bpf_fd: __u32, + pub info_len: __u32, + pub info: __u64, +} +#[repr(C)] +#[derive(Copy, Clone)] +pub struct bpf_attr__bindgen_ty_10 { + pub __bindgen_anon_1: bpf_attr__bindgen_ty_10__bindgen_ty_1, + pub attach_type: __u32, + pub query_flags: __u32, + pub attach_flags: __u32, + pub prog_ids: __u64, + pub __bindgen_anon_2: bpf_attr__bindgen_ty_10__bindgen_ty_2, + pub _bitfield_align_1: [u8; 0], + pub _bitfield_1: __BindgenBitfieldUnit<[u8; 4usize]>, + pub prog_attach_flags: __u64, + pub link_ids: __u64, + pub link_attach_flags: __u64, + pub revision: __u64, +} +#[repr(C)] +#[derive(Copy, Clone)] +pub union bpf_attr__bindgen_ty_10__bindgen_ty_1 { + pub target_fd: __u32, + pub target_ifindex: __u32, +} +#[repr(C)] +#[derive(Copy, Clone)] +pub union bpf_attr__bindgen_ty_10__bindgen_ty_2 { + pub prog_cnt: __u32, + pub count: __u32, +} +impl bpf_attr__bindgen_ty_10 { + #[inline] + pub fn new_bitfield_1() -> __BindgenBitfieldUnit<[u8; 4usize]> { + let mut __bindgen_bitfield_unit: __BindgenBitfieldUnit<[u8; 4usize]> = Default::default(); + __bindgen_bitfield_unit + } +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct bpf_attr__bindgen_ty_11 { + pub name: __u64, + pub prog_fd: __u32, + pub _bitfield_align_1: [u8; 0], + pub _bitfield_1: __BindgenBitfieldUnit<[u8; 4usize]>, + pub cookie: __u64, +} +impl bpf_attr__bindgen_ty_11 { + #[inline] + pub fn new_bitfield_1() -> __BindgenBitfieldUnit<[u8; 4usize]> { + let mut __bindgen_bitfield_unit: __BindgenBitfieldUnit<[u8; 4usize]> = Default::default(); + __bindgen_bitfield_unit + } +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct bpf_attr__bindgen_ty_12 { + pub btf: __u64, + pub btf_log_buf: __u64, + pub btf_size: __u32, + pub btf_log_size: __u32, + pub btf_log_level: __u32, + pub btf_log_true_size: __u32, + pub btf_flags: __u32, + pub btf_token_fd: __s32, +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct bpf_attr__bindgen_ty_13 { + pub pid: __u32, + pub fd: __u32, + pub flags: __u32, + pub buf_len: __u32, + pub buf: __u64, + pub prog_id: __u32, + pub fd_type: __u32, + pub probe_offset: __u64, + pub probe_addr: __u64, +} +#[repr(C)] +#[derive(Copy, Clone)] +pub struct bpf_attr__bindgen_ty_14 { + pub __bindgen_anon_1: bpf_attr__bindgen_ty_14__bindgen_ty_1, + pub __bindgen_anon_2: bpf_attr__bindgen_ty_14__bindgen_ty_2, + pub attach_type: __u32, + pub flags: __u32, + pub __bindgen_anon_3: bpf_attr__bindgen_ty_14__bindgen_ty_3, +} +#[repr(C)] +#[derive(Copy, Clone)] +pub union bpf_attr__bindgen_ty_14__bindgen_ty_1 { + pub prog_fd: __u32, + pub map_fd: __u32, +} +#[repr(C)] +#[derive(Copy, Clone)] +pub union bpf_attr__bindgen_ty_14__bindgen_ty_2 { + pub target_fd: __u32, + pub target_ifindex: __u32, +} +#[repr(C)] +#[derive(Copy, Clone)] +pub union bpf_attr__bindgen_ty_14__bindgen_ty_3 { + pub target_btf_id: __u32, + pub __bindgen_anon_1: bpf_attr__bindgen_ty_14__bindgen_ty_3__bindgen_ty_1, + pub perf_event: bpf_attr__bindgen_ty_14__bindgen_ty_3__bindgen_ty_2, + pub kprobe_multi: bpf_attr__bindgen_ty_14__bindgen_ty_3__bindgen_ty_3, + pub tracing: bpf_attr__bindgen_ty_14__bindgen_ty_3__bindgen_ty_4, + pub netfilter: bpf_attr__bindgen_ty_14__bindgen_ty_3__bindgen_ty_5, + pub tcx: bpf_attr__bindgen_ty_14__bindgen_ty_3__bindgen_ty_6, + pub uprobe_multi: bpf_attr__bindgen_ty_14__bindgen_ty_3__bindgen_ty_7, + pub netkit: bpf_attr__bindgen_ty_14__bindgen_ty_3__bindgen_ty_8, +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct bpf_attr__bindgen_ty_14__bindgen_ty_3__bindgen_ty_1 { + pub iter_info: __u64, + pub iter_info_len: __u32, +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct bpf_attr__bindgen_ty_14__bindgen_ty_3__bindgen_ty_2 { + pub bpf_cookie: __u64, +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct bpf_attr__bindgen_ty_14__bindgen_ty_3__bindgen_ty_3 { + pub flags: __u32, + pub cnt: __u32, + pub syms: __u64, + pub addrs: __u64, + pub cookies: __u64, +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct bpf_attr__bindgen_ty_14__bindgen_ty_3__bindgen_ty_4 { + pub target_btf_id: __u32, + pub cookie: __u64, +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct bpf_attr__bindgen_ty_14__bindgen_ty_3__bindgen_ty_5 { + pub pf: __u32, + pub hooknum: __u32, + pub priority: __s32, + pub flags: __u32, +} +#[repr(C)] +#[derive(Copy, Clone)] +pub struct bpf_attr__bindgen_ty_14__bindgen_ty_3__bindgen_ty_6 { + pub __bindgen_anon_1: bpf_attr__bindgen_ty_14__bindgen_ty_3__bindgen_ty_6__bindgen_ty_1, + pub expected_revision: __u64, +} +#[repr(C)] +#[derive(Copy, Clone)] +pub union bpf_attr__bindgen_ty_14__bindgen_ty_3__bindgen_ty_6__bindgen_ty_1 { + pub relative_fd: __u32, + pub relative_id: __u32, +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct bpf_attr__bindgen_ty_14__bindgen_ty_3__bindgen_ty_7 { + pub path: __u64, + pub offsets: __u64, + pub ref_ctr_offsets: __u64, + pub cookies: __u64, + pub cnt: __u32, + pub flags: __u32, + pub pid: __u32, +} +#[repr(C)] +#[derive(Copy, Clone)] +pub struct bpf_attr__bindgen_ty_14__bindgen_ty_3__bindgen_ty_8 { + pub __bindgen_anon_1: bpf_attr__bindgen_ty_14__bindgen_ty_3__bindgen_ty_8__bindgen_ty_1, + pub expected_revision: __u64, +} +#[repr(C)] +#[derive(Copy, Clone)] +pub union bpf_attr__bindgen_ty_14__bindgen_ty_3__bindgen_ty_8__bindgen_ty_1 { + pub relative_fd: __u32, + pub relative_id: __u32, +} +#[repr(C)] +#[derive(Copy, Clone)] +pub struct bpf_attr__bindgen_ty_15 { + pub link_fd: __u32, + pub __bindgen_anon_1: bpf_attr__bindgen_ty_15__bindgen_ty_1, + pub flags: __u32, + pub __bindgen_anon_2: bpf_attr__bindgen_ty_15__bindgen_ty_2, +} +#[repr(C)] +#[derive(Copy, Clone)] +pub union bpf_attr__bindgen_ty_15__bindgen_ty_1 { + pub new_prog_fd: __u32, + pub new_map_fd: __u32, +} +#[repr(C)] +#[derive(Copy, Clone)] +pub union bpf_attr__bindgen_ty_15__bindgen_ty_2 { + pub old_prog_fd: __u32, + pub old_map_fd: __u32, +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct bpf_attr__bindgen_ty_16 { + pub link_fd: __u32, +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct bpf_attr__bindgen_ty_17 { + pub type_: __u32, +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct bpf_attr__bindgen_ty_18 { + pub link_fd: __u32, + pub flags: __u32, +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct bpf_attr__bindgen_ty_19 { + pub prog_fd: __u32, + pub map_fd: __u32, + pub flags: __u32, +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct bpf_attr__bindgen_ty_20 { + pub flags: __u32, + pub bpffs_fd: __u32, +} +pub const BPF_F_RECOMPUTE_CSUM: _bindgen_ty_6 = 1; +pub const BPF_F_INVALIDATE_HASH: _bindgen_ty_6 = 2; +pub type _bindgen_ty_6 = ::core::ffi::c_uint; +pub const BPF_F_HDR_FIELD_MASK: _bindgen_ty_7 = 15; +pub type _bindgen_ty_7 = ::core::ffi::c_uint; +pub const BPF_F_PSEUDO_HDR: _bindgen_ty_8 = 16; +pub const BPF_F_MARK_MANGLED_0: _bindgen_ty_8 = 32; +pub const BPF_F_MARK_ENFORCE: _bindgen_ty_8 = 64; +pub type _bindgen_ty_8 = ::core::ffi::c_uint; +pub const BPF_F_INGRESS: _bindgen_ty_9 = 1; +pub type _bindgen_ty_9 = ::core::ffi::c_uint; +pub const BPF_F_TUNINFO_IPV6: _bindgen_ty_10 = 1; +pub type _bindgen_ty_10 = ::core::ffi::c_uint; +pub const BPF_F_SKIP_FIELD_MASK: _bindgen_ty_11 = 255; +pub const BPF_F_USER_STACK: _bindgen_ty_11 = 256; +pub const BPF_F_FAST_STACK_CMP: _bindgen_ty_11 = 512; +pub const BPF_F_REUSE_STACKID: _bindgen_ty_11 = 1024; +pub const BPF_F_USER_BUILD_ID: _bindgen_ty_11 = 2048; +pub type _bindgen_ty_11 = ::core::ffi::c_uint; +pub const BPF_F_ZERO_CSUM_TX: _bindgen_ty_12 = 2; +pub const BPF_F_DONT_FRAGMENT: _bindgen_ty_12 = 4; +pub const BPF_F_SEQ_NUMBER: _bindgen_ty_12 = 8; +pub const BPF_F_NO_TUNNEL_KEY: _bindgen_ty_12 = 16; +pub type _bindgen_ty_12 = ::core::ffi::c_uint; +pub const BPF_F_TUNINFO_FLAGS: _bindgen_ty_13 = 16; +pub type _bindgen_ty_13 = ::core::ffi::c_uint; +pub const BPF_F_INDEX_MASK: _bindgen_ty_14 = 4294967295; +pub const BPF_F_CURRENT_CPU: _bindgen_ty_14 = 4294967295; +pub const BPF_F_CTXLEN_MASK: _bindgen_ty_14 = 4503595332403200; +pub type _bindgen_ty_14 = ::core::ffi::c_ulong; +pub const BPF_F_CURRENT_NETNS: _bindgen_ty_15 = -1; +pub type _bindgen_ty_15 = ::core::ffi::c_int; +pub const BPF_F_ADJ_ROOM_FIXED_GSO: _bindgen_ty_17 = 1; +pub const BPF_F_ADJ_ROOM_ENCAP_L3_IPV4: _bindgen_ty_17 = 2; +pub const BPF_F_ADJ_ROOM_ENCAP_L3_IPV6: _bindgen_ty_17 = 4; +pub const BPF_F_ADJ_ROOM_ENCAP_L4_GRE: _bindgen_ty_17 = 8; +pub const BPF_F_ADJ_ROOM_ENCAP_L4_UDP: _bindgen_ty_17 = 16; +pub const BPF_F_ADJ_ROOM_NO_CSUM_RESET: _bindgen_ty_17 = 32; +pub const BPF_F_ADJ_ROOM_ENCAP_L2_ETH: _bindgen_ty_17 = 64; +pub const BPF_F_ADJ_ROOM_DECAP_L3_IPV4: _bindgen_ty_17 = 128; +pub const BPF_F_ADJ_ROOM_DECAP_L3_IPV6: _bindgen_ty_17 = 256; +pub type _bindgen_ty_17 = ::core::ffi::c_uint; +pub const BPF_F_SYSCTL_BASE_NAME: _bindgen_ty_19 = 1; +pub type _bindgen_ty_19 = ::core::ffi::c_uint; +pub const BPF_F_GET_BRANCH_RECORDS_SIZE: _bindgen_ty_21 = 1; +pub type _bindgen_ty_21 = ::core::ffi::c_uint; +pub const BPF_RINGBUF_BUSY_BIT: _bindgen_ty_24 = 2147483648; +pub const BPF_RINGBUF_DISCARD_BIT: _bindgen_ty_24 = 1073741824; +pub const BPF_RINGBUF_HDR_SZ: _bindgen_ty_24 = 8; +pub type _bindgen_ty_24 = ::core::ffi::c_uint; +pub const BPF_F_BPRM_SECUREEXEC: _bindgen_ty_26 = 1; +pub type _bindgen_ty_26 = ::core::ffi::c_uint; +pub const BPF_F_BROADCAST: _bindgen_ty_27 = 8; +pub const BPF_F_EXCLUDE_INGRESS: _bindgen_ty_27 = 16; +pub type _bindgen_ty_27 = ::core::ffi::c_uint; +#[repr(C)] +#[derive(Copy, Clone)] +pub struct bpf_devmap_val { + pub ifindex: __u32, + pub bpf_prog: bpf_devmap_val__bindgen_ty_1, +} +#[repr(C)] +#[derive(Copy, Clone)] +pub union bpf_devmap_val__bindgen_ty_1 { + pub fd: ::core::ffi::c_int, + pub id: __u32, +} +#[repr(C)] +#[derive(Copy, Clone)] +pub struct bpf_cpumap_val { + pub qsize: __u32, + pub bpf_prog: bpf_cpumap_val__bindgen_ty_1, +} +#[repr(C)] +#[derive(Copy, Clone)] +pub union bpf_cpumap_val__bindgen_ty_1 { + pub fd: ::core::ffi::c_int, + pub id: __u32, +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct bpf_prog_info { + pub type_: __u32, + pub id: __u32, + pub tag: [__u8; 8usize], + pub jited_prog_len: __u32, + pub xlated_prog_len: __u32, + pub jited_prog_insns: __u64, + pub xlated_prog_insns: __u64, + pub load_time: __u64, + pub created_by_uid: __u32, + pub nr_map_ids: __u32, + pub map_ids: __u64, + pub name: [::core::ffi::c_char; 16usize], + pub ifindex: __u32, + pub _bitfield_align_1: [u8; 0], + pub _bitfield_1: __BindgenBitfieldUnit<[u8; 4usize]>, + pub netns_dev: __u64, + pub netns_ino: __u64, + pub nr_jited_ksyms: __u32, + pub nr_jited_func_lens: __u32, + pub jited_ksyms: __u64, + pub jited_func_lens: __u64, + pub btf_id: __u32, + pub func_info_rec_size: __u32, + pub func_info: __u64, + pub nr_func_info: __u32, + pub nr_line_info: __u32, + pub line_info: __u64, + pub jited_line_info: __u64, + pub nr_jited_line_info: __u32, + pub line_info_rec_size: __u32, + pub jited_line_info_rec_size: __u32, + pub nr_prog_tags: __u32, + pub prog_tags: __u64, + pub run_time_ns: __u64, + pub run_cnt: __u64, + pub recursion_misses: __u64, + pub verified_insns: __u32, + pub attach_btf_obj_id: __u32, + pub attach_btf_id: __u32, +} +impl bpf_prog_info { + #[inline] + pub fn gpl_compatible(&self) -> __u32 { + unsafe { ::core::mem::transmute(self._bitfield_1.get(0usize, 1u8) as u32) } + } + #[inline] + pub fn set_gpl_compatible(&mut self, val: __u32) { + unsafe { + let val: u32 = ::core::mem::transmute(val); + self._bitfield_1.set(0usize, 1u8, val as u64) + } + } + #[inline] + pub fn new_bitfield_1(gpl_compatible: __u32) -> __BindgenBitfieldUnit<[u8; 4usize]> { + let mut __bindgen_bitfield_unit: __BindgenBitfieldUnit<[u8; 4usize]> = Default::default(); + __bindgen_bitfield_unit.set(0usize, 1u8, { + let gpl_compatible: u32 = unsafe { ::core::mem::transmute(gpl_compatible) }; + gpl_compatible as u64 + }); + __bindgen_bitfield_unit + } +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct bpf_map_info { + pub type_: __u32, + pub id: __u32, + pub key_size: __u32, + pub value_size: __u32, + pub max_entries: __u32, + pub map_flags: __u32, + pub name: [::core::ffi::c_char; 16usize], + pub ifindex: __u32, + pub btf_vmlinux_value_type_id: __u32, + pub netns_dev: __u64, + pub netns_ino: __u64, + pub btf_id: __u32, + pub btf_key_type_id: __u32, + pub btf_value_type_id: __u32, + pub btf_vmlinux_id: __u32, + pub map_extra: __u64, +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct bpf_btf_info { + pub btf: __u64, + pub btf_size: __u32, + pub id: __u32, + pub name: __u64, + pub name_len: __u32, + pub kernel_btf: __u32, +} +#[repr(C)] +#[derive(Copy, Clone)] +pub struct bpf_link_info { + pub type_: __u32, + pub id: __u32, + pub prog_id: __u32, + pub __bindgen_anon_1: bpf_link_info__bindgen_ty_1, +} +#[repr(C)] +#[derive(Copy, Clone)] +pub union bpf_link_info__bindgen_ty_1 { + pub raw_tracepoint: bpf_link_info__bindgen_ty_1__bindgen_ty_1, + pub tracing: bpf_link_info__bindgen_ty_1__bindgen_ty_2, + pub cgroup: bpf_link_info__bindgen_ty_1__bindgen_ty_3, + pub iter: bpf_link_info__bindgen_ty_1__bindgen_ty_4, + pub netns: bpf_link_info__bindgen_ty_1__bindgen_ty_5, + pub xdp: bpf_link_info__bindgen_ty_1__bindgen_ty_6, + pub struct_ops: bpf_link_info__bindgen_ty_1__bindgen_ty_7, + pub netfilter: bpf_link_info__bindgen_ty_1__bindgen_ty_8, + pub kprobe_multi: bpf_link_info__bindgen_ty_1__bindgen_ty_9, + pub uprobe_multi: bpf_link_info__bindgen_ty_1__bindgen_ty_10, + pub perf_event: bpf_link_info__bindgen_ty_1__bindgen_ty_11, + pub tcx: bpf_link_info__bindgen_ty_1__bindgen_ty_12, + pub netkit: bpf_link_info__bindgen_ty_1__bindgen_ty_13, +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct bpf_link_info__bindgen_ty_1__bindgen_ty_1 { + pub tp_name: __u64, + pub tp_name_len: __u32, +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct bpf_link_info__bindgen_ty_1__bindgen_ty_2 { + pub attach_type: __u32, + pub target_obj_id: __u32, + pub target_btf_id: __u32, +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct bpf_link_info__bindgen_ty_1__bindgen_ty_3 { + pub cgroup_id: __u64, + pub attach_type: __u32, +} +#[repr(C)] +#[derive(Copy, Clone)] +pub struct bpf_link_info__bindgen_ty_1__bindgen_ty_4 { + pub target_name: __u64, + pub target_name_len: __u32, + pub __bindgen_anon_1: bpf_link_info__bindgen_ty_1__bindgen_ty_4__bindgen_ty_1, + pub __bindgen_anon_2: bpf_link_info__bindgen_ty_1__bindgen_ty_4__bindgen_ty_2, +} +#[repr(C)] +#[derive(Copy, Clone)] +pub union bpf_link_info__bindgen_ty_1__bindgen_ty_4__bindgen_ty_1 { + pub map: bpf_link_info__bindgen_ty_1__bindgen_ty_4__bindgen_ty_1__bindgen_ty_1, +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct bpf_link_info__bindgen_ty_1__bindgen_ty_4__bindgen_ty_1__bindgen_ty_1 { + pub map_id: __u32, +} +#[repr(C)] +#[derive(Copy, Clone)] +pub union bpf_link_info__bindgen_ty_1__bindgen_ty_4__bindgen_ty_2 { + pub cgroup: bpf_link_info__bindgen_ty_1__bindgen_ty_4__bindgen_ty_2__bindgen_ty_1, + pub task: bpf_link_info__bindgen_ty_1__bindgen_ty_4__bindgen_ty_2__bindgen_ty_2, +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct bpf_link_info__bindgen_ty_1__bindgen_ty_4__bindgen_ty_2__bindgen_ty_1 { + pub cgroup_id: __u64, + pub order: __u32, +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct bpf_link_info__bindgen_ty_1__bindgen_ty_4__bindgen_ty_2__bindgen_ty_2 { + pub tid: __u32, + pub pid: __u32, +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct bpf_link_info__bindgen_ty_1__bindgen_ty_5 { + pub netns_ino: __u32, + pub attach_type: __u32, +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct bpf_link_info__bindgen_ty_1__bindgen_ty_6 { + pub ifindex: __u32, +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct bpf_link_info__bindgen_ty_1__bindgen_ty_7 { + pub map_id: __u32, +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct bpf_link_info__bindgen_ty_1__bindgen_ty_8 { + pub pf: __u32, + pub hooknum: __u32, + pub priority: __s32, + pub flags: __u32, +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct bpf_link_info__bindgen_ty_1__bindgen_ty_9 { + pub addrs: __u64, + pub count: __u32, + pub flags: __u32, + pub missed: __u64, + pub cookies: __u64, +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct bpf_link_info__bindgen_ty_1__bindgen_ty_10 { + pub path: __u64, + pub offsets: __u64, + pub ref_ctr_offsets: __u64, + pub cookies: __u64, + pub path_size: __u32, + pub count: __u32, + pub flags: __u32, + pub pid: __u32, +} +#[repr(C)] +#[derive(Copy, Clone)] +pub struct bpf_link_info__bindgen_ty_1__bindgen_ty_11 { + pub type_: __u32, + pub _bitfield_align_1: [u8; 0], + pub _bitfield_1: __BindgenBitfieldUnit<[u8; 4usize]>, + pub __bindgen_anon_1: bpf_link_info__bindgen_ty_1__bindgen_ty_11__bindgen_ty_1, +} +#[repr(C)] +#[derive(Copy, Clone)] +pub union bpf_link_info__bindgen_ty_1__bindgen_ty_11__bindgen_ty_1 { + pub uprobe: bpf_link_info__bindgen_ty_1__bindgen_ty_11__bindgen_ty_1__bindgen_ty_1, + pub kprobe: bpf_link_info__bindgen_ty_1__bindgen_ty_11__bindgen_ty_1__bindgen_ty_2, + pub tracepoint: bpf_link_info__bindgen_ty_1__bindgen_ty_11__bindgen_ty_1__bindgen_ty_3, + pub event: bpf_link_info__bindgen_ty_1__bindgen_ty_11__bindgen_ty_1__bindgen_ty_4, +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct bpf_link_info__bindgen_ty_1__bindgen_ty_11__bindgen_ty_1__bindgen_ty_1 { + pub file_name: __u64, + pub name_len: __u32, + pub offset: __u32, + pub cookie: __u64, +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct bpf_link_info__bindgen_ty_1__bindgen_ty_11__bindgen_ty_1__bindgen_ty_2 { + pub func_name: __u64, + pub name_len: __u32, + pub offset: __u32, + pub addr: __u64, + pub missed: __u64, + pub cookie: __u64, +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct bpf_link_info__bindgen_ty_1__bindgen_ty_11__bindgen_ty_1__bindgen_ty_3 { + pub tp_name: __u64, + pub name_len: __u32, + pub _bitfield_align_1: [u8; 0], + pub _bitfield_1: __BindgenBitfieldUnit<[u8; 4usize]>, + pub cookie: __u64, +} +impl bpf_link_info__bindgen_ty_1__bindgen_ty_11__bindgen_ty_1__bindgen_ty_3 { + #[inline] + pub fn new_bitfield_1() -> __BindgenBitfieldUnit<[u8; 4usize]> { + let mut __bindgen_bitfield_unit: __BindgenBitfieldUnit<[u8; 4usize]> = Default::default(); + __bindgen_bitfield_unit + } +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct bpf_link_info__bindgen_ty_1__bindgen_ty_11__bindgen_ty_1__bindgen_ty_4 { + pub config: __u64, + pub type_: __u32, + pub _bitfield_align_1: [u8; 0], + pub _bitfield_1: __BindgenBitfieldUnit<[u8; 4usize]>, + pub cookie: __u64, +} +impl bpf_link_info__bindgen_ty_1__bindgen_ty_11__bindgen_ty_1__bindgen_ty_4 { + #[inline] + pub fn new_bitfield_1() -> __BindgenBitfieldUnit<[u8; 4usize]> { + let mut __bindgen_bitfield_unit: __BindgenBitfieldUnit<[u8; 4usize]> = Default::default(); + __bindgen_bitfield_unit + } +} +impl bpf_link_info__bindgen_ty_1__bindgen_ty_11 { + #[inline] + pub fn new_bitfield_1() -> __BindgenBitfieldUnit<[u8; 4usize]> { + let mut __bindgen_bitfield_unit: __BindgenBitfieldUnit<[u8; 4usize]> = Default::default(); + __bindgen_bitfield_unit + } +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct bpf_link_info__bindgen_ty_1__bindgen_ty_12 { + pub ifindex: __u32, + pub attach_type: __u32, +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct bpf_link_info__bindgen_ty_1__bindgen_ty_13 { + pub ifindex: __u32, + pub attach_type: __u32, +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct bpf_func_info { + pub insn_off: __u32, + pub type_id: __u32, +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct bpf_line_info { + pub insn_off: __u32, + pub file_name_off: __u32, + pub line_off: __u32, + pub line_col: __u32, +} +pub const BPF_F_TIMER_ABS: _bindgen_ty_41 = 1; +pub const BPF_F_TIMER_CPU_PIN: _bindgen_ty_41 = 2; +pub type _bindgen_ty_41 = ::core::ffi::c_uint; +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct btf_header { + pub magic: __u16, + pub version: __u8, + pub flags: __u8, + pub hdr_len: __u32, + pub type_off: __u32, + pub type_len: __u32, + pub str_off: __u32, + pub str_len: __u32, +} +#[repr(C)] +#[derive(Copy, Clone)] +pub struct btf_type { + pub name_off: __u32, + pub info: __u32, + pub __bindgen_anon_1: btf_type__bindgen_ty_1, +} +#[repr(C)] +#[derive(Copy, Clone)] +pub union btf_type__bindgen_ty_1 { + pub size: __u32, + pub type_: __u32, +} +pub const BTF_KIND_UNKN: _bindgen_ty_42 = 0; +pub const BTF_KIND_INT: _bindgen_ty_42 = 1; +pub const BTF_KIND_PTR: _bindgen_ty_42 = 2; +pub const BTF_KIND_ARRAY: _bindgen_ty_42 = 3; +pub const BTF_KIND_STRUCT: _bindgen_ty_42 = 4; +pub const BTF_KIND_UNION: _bindgen_ty_42 = 5; +pub const BTF_KIND_ENUM: _bindgen_ty_42 = 6; +pub const BTF_KIND_FWD: _bindgen_ty_42 = 7; +pub const BTF_KIND_TYPEDEF: _bindgen_ty_42 = 8; +pub const BTF_KIND_VOLATILE: _bindgen_ty_42 = 9; +pub const BTF_KIND_CONST: _bindgen_ty_42 = 10; +pub const BTF_KIND_RESTRICT: _bindgen_ty_42 = 11; +pub const BTF_KIND_FUNC: _bindgen_ty_42 = 12; +pub const BTF_KIND_FUNC_PROTO: _bindgen_ty_42 = 13; +pub const BTF_KIND_VAR: _bindgen_ty_42 = 14; +pub const BTF_KIND_DATASEC: _bindgen_ty_42 = 15; +pub const BTF_KIND_FLOAT: _bindgen_ty_42 = 16; +pub const BTF_KIND_DECL_TAG: _bindgen_ty_42 = 17; +pub const BTF_KIND_TYPE_TAG: _bindgen_ty_42 = 18; +pub const BTF_KIND_ENUM64: _bindgen_ty_42 = 19; +pub const NR_BTF_KINDS: _bindgen_ty_42 = 20; +pub const BTF_KIND_MAX: _bindgen_ty_42 = 19; +pub type _bindgen_ty_42 = ::core::ffi::c_uint; +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct btf_enum { + pub name_off: __u32, + pub val: __s32, +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct btf_array { + pub type_: __u32, + pub index_type: __u32, + pub nelems: __u32, +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct btf_member { + pub name_off: __u32, + pub type_: __u32, + pub offset: __u32, +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct btf_param { + pub name_off: __u32, + pub type_: __u32, +} +pub const BTF_VAR_STATIC: _bindgen_ty_43 = 0; +pub const BTF_VAR_GLOBAL_ALLOCATED: _bindgen_ty_43 = 1; +pub const BTF_VAR_GLOBAL_EXTERN: _bindgen_ty_43 = 2; +pub type _bindgen_ty_43 = ::core::ffi::c_uint; +#[repr(u32)] +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] +pub enum btf_func_linkage { + BTF_FUNC_STATIC = 0, + BTF_FUNC_GLOBAL = 1, + BTF_FUNC_EXTERN = 2, +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct btf_var { + pub linkage: __u32, +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct btf_var_secinfo { + pub type_: __u32, + pub offset: __u32, + pub size: __u32, +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct btf_decl_tag { + pub component_idx: __s32, +} +pub const IFLA_XDP_UNSPEC: _bindgen_ty_92 = 0; +pub const IFLA_XDP_FD: _bindgen_ty_92 = 1; +pub const IFLA_XDP_ATTACHED: _bindgen_ty_92 = 2; +pub const IFLA_XDP_FLAGS: _bindgen_ty_92 = 3; +pub const IFLA_XDP_PROG_ID: _bindgen_ty_92 = 4; +pub const IFLA_XDP_DRV_PROG_ID: _bindgen_ty_92 = 5; +pub const IFLA_XDP_SKB_PROG_ID: _bindgen_ty_92 = 6; +pub const IFLA_XDP_HW_PROG_ID: _bindgen_ty_92 = 7; +pub const IFLA_XDP_EXPECTED_FD: _bindgen_ty_92 = 8; +pub const __IFLA_XDP_MAX: _bindgen_ty_92 = 9; +pub type _bindgen_ty_92 = ::core::ffi::c_uint; +#[repr(u32)] +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq, FromPrimitive)] +pub enum perf_type_id { + PERF_TYPE_HARDWARE = 0, + PERF_TYPE_SOFTWARE = 1, + PERF_TYPE_TRACEPOINT = 2, + PERF_TYPE_HW_CACHE = 3, + PERF_TYPE_RAW = 4, + PERF_TYPE_BREAKPOINT = 5, + PERF_TYPE_MAX = 6, +} +#[repr(u32)] +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] +pub enum perf_hw_id { + PERF_COUNT_HW_CPU_CYCLES = 0, + PERF_COUNT_HW_INSTRUCTIONS = 1, + PERF_COUNT_HW_CACHE_REFERENCES = 2, + PERF_COUNT_HW_CACHE_MISSES = 3, + PERF_COUNT_HW_BRANCH_INSTRUCTIONS = 4, + PERF_COUNT_HW_BRANCH_MISSES = 5, + PERF_COUNT_HW_BUS_CYCLES = 6, + PERF_COUNT_HW_STALLED_CYCLES_FRONTEND = 7, + PERF_COUNT_HW_STALLED_CYCLES_BACKEND = 8, + PERF_COUNT_HW_REF_CPU_CYCLES = 9, + PERF_COUNT_HW_MAX = 10, +} +#[repr(u32)] +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] +pub enum perf_hw_cache_id { + PERF_COUNT_HW_CACHE_L1D = 0, + PERF_COUNT_HW_CACHE_L1I = 1, + PERF_COUNT_HW_CACHE_LL = 2, + PERF_COUNT_HW_CACHE_DTLB = 3, + PERF_COUNT_HW_CACHE_ITLB = 4, + PERF_COUNT_HW_CACHE_BPU = 5, + PERF_COUNT_HW_CACHE_NODE = 6, + PERF_COUNT_HW_CACHE_MAX = 7, +} +#[repr(u32)] +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] +pub enum perf_hw_cache_op_id { + PERF_COUNT_HW_CACHE_OP_READ = 0, + PERF_COUNT_HW_CACHE_OP_WRITE = 1, + PERF_COUNT_HW_CACHE_OP_PREFETCH = 2, + PERF_COUNT_HW_CACHE_OP_MAX = 3, +} +#[repr(u32)] +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] +pub enum perf_hw_cache_op_result_id { + PERF_COUNT_HW_CACHE_RESULT_ACCESS = 0, + PERF_COUNT_HW_CACHE_RESULT_MISS = 1, + PERF_COUNT_HW_CACHE_RESULT_MAX = 2, +} +#[repr(u32)] +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq, FromPrimitive)] +pub enum perf_sw_ids { + PERF_COUNT_SW_CPU_CLOCK = 0, + PERF_COUNT_SW_TASK_CLOCK = 1, + PERF_COUNT_SW_PAGE_FAULTS = 2, + PERF_COUNT_SW_CONTEXT_SWITCHES = 3, + PERF_COUNT_SW_CPU_MIGRATIONS = 4, + PERF_COUNT_SW_PAGE_FAULTS_MIN = 5, + PERF_COUNT_SW_PAGE_FAULTS_MAJ = 6, + PERF_COUNT_SW_ALIGNMENT_FAULTS = 7, + PERF_COUNT_SW_EMULATION_FAULTS = 8, + PERF_COUNT_SW_DUMMY = 9, + PERF_COUNT_SW_BPF_OUTPUT = 10, + PERF_COUNT_SW_CGROUP_SWITCHES = 11, + PERF_COUNT_SW_MAX = 12, +} +#[repr(u32)] +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq, FromPrimitive)] +pub enum perf_event_sample_format { + PERF_SAMPLE_IP = 1, + PERF_SAMPLE_TID = 2, + PERF_SAMPLE_TIME = 4, + PERF_SAMPLE_ADDR = 8, + PERF_SAMPLE_READ = 16, + PERF_SAMPLE_CALLCHAIN = 32, + PERF_SAMPLE_ID = 64, + PERF_SAMPLE_CPU = 128, + PERF_SAMPLE_PERIOD = 256, + PERF_SAMPLE_STREAM_ID = 512, + PERF_SAMPLE_RAW = 1024, + PERF_SAMPLE_BRANCH_STACK = 2048, + PERF_SAMPLE_REGS_USER = 4096, + PERF_SAMPLE_STACK_USER = 8192, + PERF_SAMPLE_WEIGHT = 16384, + PERF_SAMPLE_DATA_SRC = 32768, + PERF_SAMPLE_IDENTIFIER = 65536, + PERF_SAMPLE_TRANSACTION = 131072, + PERF_SAMPLE_REGS_INTR = 262144, + PERF_SAMPLE_PHYS_ADDR = 524288, + PERF_SAMPLE_AUX = 1048576, + PERF_SAMPLE_CGROUP = 2097152, + PERF_SAMPLE_DATA_PAGE_SIZE = 4194304, + PERF_SAMPLE_CODE_PAGE_SIZE = 8388608, + PERF_SAMPLE_WEIGHT_STRUCT = 16777216, + PERF_SAMPLE_MAX = 33554432, +} +#[repr(C)] +#[derive(Copy, Clone)] +pub struct perf_event_attr { + pub type_: __u32, + pub size: __u32, + pub config: __u64, + pub __bindgen_anon_1: perf_event_attr__bindgen_ty_1, + pub sample_type: __u64, + pub read_format: __u64, + pub _bitfield_align_1: [u32; 0], + pub _bitfield_1: __BindgenBitfieldUnit<[u8; 8usize]>, + pub __bindgen_anon_2: perf_event_attr__bindgen_ty_2, + pub bp_type: __u32, + pub __bindgen_anon_3: perf_event_attr__bindgen_ty_3, + pub __bindgen_anon_4: perf_event_attr__bindgen_ty_4, + pub branch_sample_type: __u64, + pub sample_regs_user: __u64, + pub sample_stack_user: __u32, + pub clockid: __s32, + pub sample_regs_intr: __u64, + pub aux_watermark: __u32, + pub sample_max_stack: __u16, + pub __reserved_2: __u16, + pub aux_sample_size: __u32, + pub __reserved_3: __u32, + pub sig_data: __u64, + pub config3: __u64, +} +#[repr(C)] +#[derive(Copy, Clone)] +pub union perf_event_attr__bindgen_ty_1 { + pub sample_period: __u64, + pub sample_freq: __u64, +} +#[repr(C)] +#[derive(Copy, Clone)] +pub union perf_event_attr__bindgen_ty_2 { + pub wakeup_events: __u32, + pub wakeup_watermark: __u32, +} +#[repr(C)] +#[derive(Copy, Clone)] +pub union perf_event_attr__bindgen_ty_3 { + pub bp_addr: __u64, + pub kprobe_func: __u64, + pub uprobe_path: __u64, + pub config1: __u64, +} +#[repr(C)] +#[derive(Copy, Clone)] +pub union perf_event_attr__bindgen_ty_4 { + pub bp_len: __u64, + pub kprobe_addr: __u64, + pub probe_offset: __u64, + pub config2: __u64, +} +impl perf_event_attr { + #[inline] + pub fn disabled(&self) -> __u64 { + unsafe { ::core::mem::transmute(self._bitfield_1.get(0usize, 1u8) as u64) } + } + #[inline] + pub fn set_disabled(&mut self, val: __u64) { + unsafe { + let val: u64 = ::core::mem::transmute(val); + self._bitfield_1.set(0usize, 1u8, val as u64) + } + } + #[inline] + pub fn inherit(&self) -> __u64 { + unsafe { ::core::mem::transmute(self._bitfield_1.get(1usize, 1u8) as u64) } + } + #[inline] + pub fn set_inherit(&mut self, val: __u64) { + unsafe { + let val: u64 = ::core::mem::transmute(val); + self._bitfield_1.set(1usize, 1u8, val as u64) + } + } + #[inline] + pub fn pinned(&self) -> __u64 { + unsafe { ::core::mem::transmute(self._bitfield_1.get(2usize, 1u8) as u64) } + } + #[inline] + pub fn set_pinned(&mut self, val: __u64) { + unsafe { + let val: u64 = ::core::mem::transmute(val); + self._bitfield_1.set(2usize, 1u8, val as u64) + } + } + #[inline] + pub fn exclusive(&self) -> __u64 { + unsafe { ::core::mem::transmute(self._bitfield_1.get(3usize, 1u8) as u64) } + } + #[inline] + pub fn set_exclusive(&mut self, val: __u64) { + unsafe { + let val: u64 = ::core::mem::transmute(val); + self._bitfield_1.set(3usize, 1u8, val as u64) + } + } + #[inline] + pub fn exclude_user(&self) -> __u64 { + unsafe { ::core::mem::transmute(self._bitfield_1.get(4usize, 1u8) as u64) } + } + #[inline] + pub fn set_exclude_user(&mut self, val: __u64) { + unsafe { + let val: u64 = ::core::mem::transmute(val); + self._bitfield_1.set(4usize, 1u8, val as u64) + } + } + #[inline] + pub fn exclude_kernel(&self) -> __u64 { + unsafe { ::core::mem::transmute(self._bitfield_1.get(5usize, 1u8) as u64) } + } + #[inline] + pub fn set_exclude_kernel(&mut self, val: __u64) { + unsafe { + let val: u64 = ::core::mem::transmute(val); + self._bitfield_1.set(5usize, 1u8, val as u64) + } + } + #[inline] + pub fn exclude_hv(&self) -> __u64 { + unsafe { ::core::mem::transmute(self._bitfield_1.get(6usize, 1u8) as u64) } + } + #[inline] + pub fn set_exclude_hv(&mut self, val: __u64) { + unsafe { + let val: u64 = ::core::mem::transmute(val); + self._bitfield_1.set(6usize, 1u8, val as u64) + } + } + #[inline] + pub fn exclude_idle(&self) -> __u64 { + unsafe { ::core::mem::transmute(self._bitfield_1.get(7usize, 1u8) as u64) } + } + #[inline] + pub fn set_exclude_idle(&mut self, val: __u64) { + unsafe { + let val: u64 = ::core::mem::transmute(val); + self._bitfield_1.set(7usize, 1u8, val as u64) + } + } + #[inline] + pub fn mmap(&self) -> __u64 { + unsafe { ::core::mem::transmute(self._bitfield_1.get(8usize, 1u8) as u64) } + } + #[inline] + pub fn set_mmap(&mut self, val: __u64) { + unsafe { + let val: u64 = ::core::mem::transmute(val); + self._bitfield_1.set(8usize, 1u8, val as u64) + } + } + #[inline] + pub fn comm(&self) -> __u64 { + unsafe { ::core::mem::transmute(self._bitfield_1.get(9usize, 1u8) as u64) } + } + #[inline] + pub fn set_comm(&mut self, val: __u64) { + unsafe { + let val: u64 = ::core::mem::transmute(val); + self._bitfield_1.set(9usize, 1u8, val as u64) + } + } + #[inline] + pub fn freq(&self) -> __u64 { + unsafe { ::core::mem::transmute(self._bitfield_1.get(10usize, 1u8) as u64) } + } + #[inline] + pub fn set_freq(&mut self, val: __u64) { + unsafe { + let val: u64 = ::core::mem::transmute(val); + self._bitfield_1.set(10usize, 1u8, val as u64) + } + } + #[inline] + pub fn inherit_stat(&self) -> __u64 { + unsafe { ::core::mem::transmute(self._bitfield_1.get(11usize, 1u8) as u64) } + } + #[inline] + pub fn set_inherit_stat(&mut self, val: __u64) { + unsafe { + let val: u64 = ::core::mem::transmute(val); + self._bitfield_1.set(11usize, 1u8, val as u64) + } + } + #[inline] + pub fn enable_on_exec(&self) -> __u64 { + unsafe { ::core::mem::transmute(self._bitfield_1.get(12usize, 1u8) as u64) } + } + #[inline] + pub fn set_enable_on_exec(&mut self, val: __u64) { + unsafe { + let val: u64 = ::core::mem::transmute(val); + self._bitfield_1.set(12usize, 1u8, val as u64) + } + } + #[inline] + pub fn task(&self) -> __u64 { + unsafe { ::core::mem::transmute(self._bitfield_1.get(13usize, 1u8) as u64) } + } + #[inline] + pub fn set_task(&mut self, val: __u64) { + unsafe { + let val: u64 = ::core::mem::transmute(val); + self._bitfield_1.set(13usize, 1u8, val as u64) + } + } + #[inline] + pub fn watermark(&self) -> __u64 { + unsafe { ::core::mem::transmute(self._bitfield_1.get(14usize, 1u8) as u64) } + } + #[inline] + pub fn set_watermark(&mut self, val: __u64) { + unsafe { + let val: u64 = ::core::mem::transmute(val); + self._bitfield_1.set(14usize, 1u8, val as u64) + } + } + #[inline] + pub fn precise_ip(&self) -> __u64 { + unsafe { ::core::mem::transmute(self._bitfield_1.get(15usize, 2u8) as u64) } + } + #[inline] + pub fn set_precise_ip(&mut self, val: __u64) { + unsafe { + let val: u64 = ::core::mem::transmute(val); + self._bitfield_1.set(15usize, 2u8, val as u64) + } + } + #[inline] + pub fn mmap_data(&self) -> __u64 { + unsafe { ::core::mem::transmute(self._bitfield_1.get(17usize, 1u8) as u64) } + } + #[inline] + pub fn set_mmap_data(&mut self, val: __u64) { + unsafe { + let val: u64 = ::core::mem::transmute(val); + self._bitfield_1.set(17usize, 1u8, val as u64) + } + } + #[inline] + pub fn sample_id_all(&self) -> __u64 { + unsafe { ::core::mem::transmute(self._bitfield_1.get(18usize, 1u8) as u64) } + } + #[inline] + pub fn set_sample_id_all(&mut self, val: __u64) { + unsafe { + let val: u64 = ::core::mem::transmute(val); + self._bitfield_1.set(18usize, 1u8, val as u64) + } + } + #[inline] + pub fn exclude_host(&self) -> __u64 { + unsafe { ::core::mem::transmute(self._bitfield_1.get(19usize, 1u8) as u64) } + } + #[inline] + pub fn set_exclude_host(&mut self, val: __u64) { + unsafe { + let val: u64 = ::core::mem::transmute(val); + self._bitfield_1.set(19usize, 1u8, val as u64) + } + } + #[inline] + pub fn exclude_guest(&self) -> __u64 { + unsafe { ::core::mem::transmute(self._bitfield_1.get(20usize, 1u8) as u64) } + } + #[inline] + pub fn set_exclude_guest(&mut self, val: __u64) { + unsafe { + let val: u64 = ::core::mem::transmute(val); + self._bitfield_1.set(20usize, 1u8, val as u64) + } + } + #[inline] + pub fn exclude_callchain_kernel(&self) -> __u64 { + unsafe { ::core::mem::transmute(self._bitfield_1.get(21usize, 1u8) as u64) } + } + #[inline] + pub fn set_exclude_callchain_kernel(&mut self, val: __u64) { + unsafe { + let val: u64 = ::core::mem::transmute(val); + self._bitfield_1.set(21usize, 1u8, val as u64) + } + } + #[inline] + pub fn exclude_callchain_user(&self) -> __u64 { + unsafe { ::core::mem::transmute(self._bitfield_1.get(22usize, 1u8) as u64) } + } + #[inline] + pub fn set_exclude_callchain_user(&mut self, val: __u64) { + unsafe { + let val: u64 = ::core::mem::transmute(val); + self._bitfield_1.set(22usize, 1u8, val as u64) + } + } + #[inline] + pub fn mmap2(&self) -> __u64 { + unsafe { ::core::mem::transmute(self._bitfield_1.get(23usize, 1u8) as u64) } + } + #[inline] + pub fn set_mmap2(&mut self, val: __u64) { + unsafe { + let val: u64 = ::core::mem::transmute(val); + self._bitfield_1.set(23usize, 1u8, val as u64) + } + } + #[inline] + pub fn comm_exec(&self) -> __u64 { + unsafe { ::core::mem::transmute(self._bitfield_1.get(24usize, 1u8) as u64) } + } + #[inline] + pub fn set_comm_exec(&mut self, val: __u64) { + unsafe { + let val: u64 = ::core::mem::transmute(val); + self._bitfield_1.set(24usize, 1u8, val as u64) + } + } + #[inline] + pub fn use_clockid(&self) -> __u64 { + unsafe { ::core::mem::transmute(self._bitfield_1.get(25usize, 1u8) as u64) } + } + #[inline] + pub fn set_use_clockid(&mut self, val: __u64) { + unsafe { + let val: u64 = ::core::mem::transmute(val); + self._bitfield_1.set(25usize, 1u8, val as u64) + } + } + #[inline] + pub fn context_switch(&self) -> __u64 { + unsafe { ::core::mem::transmute(self._bitfield_1.get(26usize, 1u8) as u64) } + } + #[inline] + pub fn set_context_switch(&mut self, val: __u64) { + unsafe { + let val: u64 = ::core::mem::transmute(val); + self._bitfield_1.set(26usize, 1u8, val as u64) + } + } + #[inline] + pub fn write_backward(&self) -> __u64 { + unsafe { ::core::mem::transmute(self._bitfield_1.get(27usize, 1u8) as u64) } + } + #[inline] + pub fn set_write_backward(&mut self, val: __u64) { + unsafe { + let val: u64 = ::core::mem::transmute(val); + self._bitfield_1.set(27usize, 1u8, val as u64) + } + } + #[inline] + pub fn namespaces(&self) -> __u64 { + unsafe { ::core::mem::transmute(self._bitfield_1.get(28usize, 1u8) as u64) } + } + #[inline] + pub fn set_namespaces(&mut self, val: __u64) { + unsafe { + let val: u64 = ::core::mem::transmute(val); + self._bitfield_1.set(28usize, 1u8, val as u64) + } + } + #[inline] + pub fn ksymbol(&self) -> __u64 { + unsafe { ::core::mem::transmute(self._bitfield_1.get(29usize, 1u8) as u64) } + } + #[inline] + pub fn set_ksymbol(&mut self, val: __u64) { + unsafe { + let val: u64 = ::core::mem::transmute(val); + self._bitfield_1.set(29usize, 1u8, val as u64) + } + } + #[inline] + pub fn bpf_event(&self) -> __u64 { + unsafe { ::core::mem::transmute(self._bitfield_1.get(30usize, 1u8) as u64) } + } + #[inline] + pub fn set_bpf_event(&mut self, val: __u64) { + unsafe { + let val: u64 = ::core::mem::transmute(val); + self._bitfield_1.set(30usize, 1u8, val as u64) + } + } + #[inline] + pub fn aux_output(&self) -> __u64 { + unsafe { ::core::mem::transmute(self._bitfield_1.get(31usize, 1u8) as u64) } + } + #[inline] + pub fn set_aux_output(&mut self, val: __u64) { + unsafe { + let val: u64 = ::core::mem::transmute(val); + self._bitfield_1.set(31usize, 1u8, val as u64) + } + } + #[inline] + pub fn cgroup(&self) -> __u64 { + unsafe { ::core::mem::transmute(self._bitfield_1.get(32usize, 1u8) as u64) } + } + #[inline] + pub fn set_cgroup(&mut self, val: __u64) { + unsafe { + let val: u64 = ::core::mem::transmute(val); + self._bitfield_1.set(32usize, 1u8, val as u64) + } + } + #[inline] + pub fn text_poke(&self) -> __u64 { + unsafe { ::core::mem::transmute(self._bitfield_1.get(33usize, 1u8) as u64) } + } + #[inline] + pub fn set_text_poke(&mut self, val: __u64) { + unsafe { + let val: u64 = ::core::mem::transmute(val); + self._bitfield_1.set(33usize, 1u8, val as u64) + } + } + #[inline] + pub fn build_id(&self) -> __u64 { + unsafe { ::core::mem::transmute(self._bitfield_1.get(34usize, 1u8) as u64) } + } + #[inline] + pub fn set_build_id(&mut self, val: __u64) { + unsafe { + let val: u64 = ::core::mem::transmute(val); + self._bitfield_1.set(34usize, 1u8, val as u64) + } + } + #[inline] + pub fn inherit_thread(&self) -> __u64 { + unsafe { ::core::mem::transmute(self._bitfield_1.get(35usize, 1u8) as u64) } + } + #[inline] + pub fn set_inherit_thread(&mut self, val: __u64) { + unsafe { + let val: u64 = ::core::mem::transmute(val); + self._bitfield_1.set(35usize, 1u8, val as u64) + } + } + #[inline] + pub fn remove_on_exec(&self) -> __u64 { + unsafe { ::core::mem::transmute(self._bitfield_1.get(36usize, 1u8) as u64) } + } + #[inline] + pub fn set_remove_on_exec(&mut self, val: __u64) { + unsafe { + let val: u64 = ::core::mem::transmute(val); + self._bitfield_1.set(36usize, 1u8, val as u64) + } + } + #[inline] + pub fn sigtrap(&self) -> __u64 { + unsafe { ::core::mem::transmute(self._bitfield_1.get(37usize, 1u8) as u64) } + } + #[inline] + pub fn set_sigtrap(&mut self, val: __u64) { + unsafe { + let val: u64 = ::core::mem::transmute(val); + self._bitfield_1.set(37usize, 1u8, val as u64) + } + } + #[inline] + pub fn __reserved_1(&self) -> __u64 { + unsafe { ::core::mem::transmute(self._bitfield_1.get(38usize, 26u8) as u64) } + } + #[inline] + pub fn set___reserved_1(&mut self, val: __u64) { + unsafe { + let val: u64 = ::core::mem::transmute(val); + self._bitfield_1.set(38usize, 26u8, val as u64) + } + } + #[inline] + pub fn new_bitfield_1( + disabled: __u64, + inherit: __u64, + pinned: __u64, + exclusive: __u64, + exclude_user: __u64, + exclude_kernel: __u64, + exclude_hv: __u64, + exclude_idle: __u64, + mmap: __u64, + comm: __u64, + freq: __u64, + inherit_stat: __u64, + enable_on_exec: __u64, + task: __u64, + watermark: __u64, + precise_ip: __u64, + mmap_data: __u64, + sample_id_all: __u64, + exclude_host: __u64, + exclude_guest: __u64, + exclude_callchain_kernel: __u64, + exclude_callchain_user: __u64, + mmap2: __u64, + comm_exec: __u64, + use_clockid: __u64, + context_switch: __u64, + write_backward: __u64, + namespaces: __u64, + ksymbol: __u64, + bpf_event: __u64, + aux_output: __u64, + cgroup: __u64, + text_poke: __u64, + build_id: __u64, + inherit_thread: __u64, + remove_on_exec: __u64, + sigtrap: __u64, + __reserved_1: __u64, + ) -> __BindgenBitfieldUnit<[u8; 8usize]> { + let mut __bindgen_bitfield_unit: __BindgenBitfieldUnit<[u8; 8usize]> = Default::default(); + __bindgen_bitfield_unit.set(0usize, 1u8, { + let disabled: u64 = unsafe { ::core::mem::transmute(disabled) }; + disabled as u64 + }); + __bindgen_bitfield_unit.set(1usize, 1u8, { + let inherit: u64 = unsafe { ::core::mem::transmute(inherit) }; + inherit as u64 + }); + __bindgen_bitfield_unit.set(2usize, 1u8, { + let pinned: u64 = unsafe { ::core::mem::transmute(pinned) }; + pinned as u64 + }); + __bindgen_bitfield_unit.set(3usize, 1u8, { + let exclusive: u64 = unsafe { ::core::mem::transmute(exclusive) }; + exclusive as u64 + }); + __bindgen_bitfield_unit.set(4usize, 1u8, { + let exclude_user: u64 = unsafe { ::core::mem::transmute(exclude_user) }; + exclude_user as u64 + }); + __bindgen_bitfield_unit.set(5usize, 1u8, { + let exclude_kernel: u64 = unsafe { ::core::mem::transmute(exclude_kernel) }; + exclude_kernel as u64 + }); + __bindgen_bitfield_unit.set(6usize, 1u8, { + let exclude_hv: u64 = unsafe { ::core::mem::transmute(exclude_hv) }; + exclude_hv as u64 + }); + __bindgen_bitfield_unit.set(7usize, 1u8, { + let exclude_idle: u64 = unsafe { ::core::mem::transmute(exclude_idle) }; + exclude_idle as u64 + }); + __bindgen_bitfield_unit.set(8usize, 1u8, { + let mmap: u64 = unsafe { ::core::mem::transmute(mmap) }; + mmap as u64 + }); + __bindgen_bitfield_unit.set(9usize, 1u8, { + let comm: u64 = unsafe { ::core::mem::transmute(comm) }; + comm as u64 + }); + __bindgen_bitfield_unit.set(10usize, 1u8, { + let freq: u64 = unsafe { ::core::mem::transmute(freq) }; + freq as u64 + }); + __bindgen_bitfield_unit.set(11usize, 1u8, { + let inherit_stat: u64 = unsafe { ::core::mem::transmute(inherit_stat) }; + inherit_stat as u64 + }); + __bindgen_bitfield_unit.set(12usize, 1u8, { + let enable_on_exec: u64 = unsafe { ::core::mem::transmute(enable_on_exec) }; + enable_on_exec as u64 + }); + __bindgen_bitfield_unit.set(13usize, 1u8, { + let task: u64 = unsafe { ::core::mem::transmute(task) }; + task as u64 + }); + __bindgen_bitfield_unit.set(14usize, 1u8, { + let watermark: u64 = unsafe { ::core::mem::transmute(watermark) }; + watermark as u64 + }); + __bindgen_bitfield_unit.set(15usize, 2u8, { + let precise_ip: u64 = unsafe { ::core::mem::transmute(precise_ip) }; + precise_ip as u64 + }); + __bindgen_bitfield_unit.set(17usize, 1u8, { + let mmap_data: u64 = unsafe { ::core::mem::transmute(mmap_data) }; + mmap_data as u64 + }); + __bindgen_bitfield_unit.set(18usize, 1u8, { + let sample_id_all: u64 = unsafe { ::core::mem::transmute(sample_id_all) }; + sample_id_all as u64 + }); + __bindgen_bitfield_unit.set(19usize, 1u8, { + let exclude_host: u64 = unsafe { ::core::mem::transmute(exclude_host) }; + exclude_host as u64 + }); + __bindgen_bitfield_unit.set(20usize, 1u8, { + let exclude_guest: u64 = unsafe { ::core::mem::transmute(exclude_guest) }; + exclude_guest as u64 + }); + __bindgen_bitfield_unit.set(21usize, 1u8, { + let exclude_callchain_kernel: u64 = + unsafe { ::core::mem::transmute(exclude_callchain_kernel) }; + exclude_callchain_kernel as u64 + }); + __bindgen_bitfield_unit.set(22usize, 1u8, { + let exclude_callchain_user: u64 = + unsafe { ::core::mem::transmute(exclude_callchain_user) }; + exclude_callchain_user as u64 + }); + __bindgen_bitfield_unit.set(23usize, 1u8, { + let mmap2: u64 = unsafe { ::core::mem::transmute(mmap2) }; + mmap2 as u64 + }); + __bindgen_bitfield_unit.set(24usize, 1u8, { + let comm_exec: u64 = unsafe { ::core::mem::transmute(comm_exec) }; + comm_exec as u64 + }); + __bindgen_bitfield_unit.set(25usize, 1u8, { + let use_clockid: u64 = unsafe { ::core::mem::transmute(use_clockid) }; + use_clockid as u64 + }); + __bindgen_bitfield_unit.set(26usize, 1u8, { + let context_switch: u64 = unsafe { ::core::mem::transmute(context_switch) }; + context_switch as u64 + }); + __bindgen_bitfield_unit.set(27usize, 1u8, { + let write_backward: u64 = unsafe { ::core::mem::transmute(write_backward) }; + write_backward as u64 + }); + __bindgen_bitfield_unit.set(28usize, 1u8, { + let namespaces: u64 = unsafe { ::core::mem::transmute(namespaces) }; + namespaces as u64 + }); + __bindgen_bitfield_unit.set(29usize, 1u8, { + let ksymbol: u64 = unsafe { ::core::mem::transmute(ksymbol) }; + ksymbol as u64 + }); + __bindgen_bitfield_unit.set(30usize, 1u8, { + let bpf_event: u64 = unsafe { ::core::mem::transmute(bpf_event) }; + bpf_event as u64 + }); + __bindgen_bitfield_unit.set(31usize, 1u8, { + let aux_output: u64 = unsafe { ::core::mem::transmute(aux_output) }; + aux_output as u64 + }); + __bindgen_bitfield_unit.set(32usize, 1u8, { + let cgroup: u64 = unsafe { ::core::mem::transmute(cgroup) }; + cgroup as u64 + }); + __bindgen_bitfield_unit.set(33usize, 1u8, { + let text_poke: u64 = unsafe { ::core::mem::transmute(text_poke) }; + text_poke as u64 + }); + __bindgen_bitfield_unit.set(34usize, 1u8, { + let build_id: u64 = unsafe { ::core::mem::transmute(build_id) }; + build_id as u64 + }); + __bindgen_bitfield_unit.set(35usize, 1u8, { + let inherit_thread: u64 = unsafe { ::core::mem::transmute(inherit_thread) }; + inherit_thread as u64 + }); + __bindgen_bitfield_unit.set(36usize, 1u8, { + let remove_on_exec: u64 = unsafe { ::core::mem::transmute(remove_on_exec) }; + remove_on_exec as u64 + }); + __bindgen_bitfield_unit.set(37usize, 1u8, { + let sigtrap: u64 = unsafe { ::core::mem::transmute(sigtrap) }; + sigtrap as u64 + }); + __bindgen_bitfield_unit.set(38usize, 26u8, { + let __reserved_1: u64 = unsafe { ::core::mem::transmute(__reserved_1) }; + __reserved_1 as u64 + }); + __bindgen_bitfield_unit + } +} +#[repr(C)] +#[derive(Copy, Clone)] +pub struct perf_event_mmap_page { + pub version: __u32, + pub compat_version: __u32, + pub lock: __u32, + pub index: __u32, + pub offset: __s64, + pub time_enabled: __u64, + pub time_running: __u64, + pub __bindgen_anon_1: perf_event_mmap_page__bindgen_ty_1, + pub pmc_width: __u16, + pub time_shift: __u16, + pub time_mult: __u32, + pub time_offset: __u64, + pub time_zero: __u64, + pub size: __u32, + pub __reserved_1: __u32, + pub time_cycles: __u64, + pub time_mask: __u64, + pub __reserved: [__u8; 928usize], + pub data_head: __u64, + pub data_tail: __u64, + pub data_offset: __u64, + pub data_size: __u64, + pub aux_head: __u64, + pub aux_tail: __u64, + pub aux_offset: __u64, + pub aux_size: __u64, +} +#[repr(C)] +#[derive(Copy, Clone)] +pub union perf_event_mmap_page__bindgen_ty_1 { + pub capabilities: __u64, + pub __bindgen_anon_1: perf_event_mmap_page__bindgen_ty_1__bindgen_ty_1, +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct perf_event_mmap_page__bindgen_ty_1__bindgen_ty_1 { + pub _bitfield_align_1: [u64; 0], + pub _bitfield_1: __BindgenBitfieldUnit<[u8; 8usize]>, +} +impl perf_event_mmap_page__bindgen_ty_1__bindgen_ty_1 { + #[inline] + pub fn cap_bit0(&self) -> __u64 { + unsafe { ::core::mem::transmute(self._bitfield_1.get(0usize, 1u8) as u64) } + } + #[inline] + pub fn set_cap_bit0(&mut self, val: __u64) { + unsafe { + let val: u64 = ::core::mem::transmute(val); + self._bitfield_1.set(0usize, 1u8, val as u64) + } + } + #[inline] + pub fn cap_bit0_is_deprecated(&self) -> __u64 { + unsafe { ::core::mem::transmute(self._bitfield_1.get(1usize, 1u8) as u64) } + } + #[inline] + pub fn set_cap_bit0_is_deprecated(&mut self, val: __u64) { + unsafe { + let val: u64 = ::core::mem::transmute(val); + self._bitfield_1.set(1usize, 1u8, val as u64) + } + } + #[inline] + pub fn cap_user_rdpmc(&self) -> __u64 { + unsafe { ::core::mem::transmute(self._bitfield_1.get(2usize, 1u8) as u64) } + } + #[inline] + pub fn set_cap_user_rdpmc(&mut self, val: __u64) { + unsafe { + let val: u64 = ::core::mem::transmute(val); + self._bitfield_1.set(2usize, 1u8, val as u64) + } + } + #[inline] + pub fn cap_user_time(&self) -> __u64 { + unsafe { ::core::mem::transmute(self._bitfield_1.get(3usize, 1u8) as u64) } + } + #[inline] + pub fn set_cap_user_time(&mut self, val: __u64) { + unsafe { + let val: u64 = ::core::mem::transmute(val); + self._bitfield_1.set(3usize, 1u8, val as u64) + } + } + #[inline] + pub fn cap_user_time_zero(&self) -> __u64 { + unsafe { ::core::mem::transmute(self._bitfield_1.get(4usize, 1u8) as u64) } + } + #[inline] + pub fn set_cap_user_time_zero(&mut self, val: __u64) { + unsafe { + let val: u64 = ::core::mem::transmute(val); + self._bitfield_1.set(4usize, 1u8, val as u64) + } + } + #[inline] + pub fn cap_user_time_short(&self) -> __u64 { + unsafe { ::core::mem::transmute(self._bitfield_1.get(5usize, 1u8) as u64) } + } + #[inline] + pub fn set_cap_user_time_short(&mut self, val: __u64) { + unsafe { + let val: u64 = ::core::mem::transmute(val); + self._bitfield_1.set(5usize, 1u8, val as u64) + } + } + #[inline] + pub fn cap_____res(&self) -> __u64 { + unsafe { ::core::mem::transmute(self._bitfield_1.get(6usize, 58u8) as u64) } + } + #[inline] + pub fn set_cap_____res(&mut self, val: __u64) { + unsafe { + let val: u64 = ::core::mem::transmute(val); + self._bitfield_1.set(6usize, 58u8, val as u64) + } + } + #[inline] + pub fn new_bitfield_1( + cap_bit0: __u64, + cap_bit0_is_deprecated: __u64, + cap_user_rdpmc: __u64, + cap_user_time: __u64, + cap_user_time_zero: __u64, + cap_user_time_short: __u64, + cap_____res: __u64, + ) -> __BindgenBitfieldUnit<[u8; 8usize]> { + let mut __bindgen_bitfield_unit: __BindgenBitfieldUnit<[u8; 8usize]> = Default::default(); + __bindgen_bitfield_unit.set(0usize, 1u8, { + let cap_bit0: u64 = unsafe { ::core::mem::transmute(cap_bit0) }; + cap_bit0 as u64 + }); + __bindgen_bitfield_unit.set(1usize, 1u8, { + let cap_bit0_is_deprecated: u64 = + unsafe { ::core::mem::transmute(cap_bit0_is_deprecated) }; + cap_bit0_is_deprecated as u64 + }); + __bindgen_bitfield_unit.set(2usize, 1u8, { + let cap_user_rdpmc: u64 = unsafe { ::core::mem::transmute(cap_user_rdpmc) }; + cap_user_rdpmc as u64 + }); + __bindgen_bitfield_unit.set(3usize, 1u8, { + let cap_user_time: u64 = unsafe { ::core::mem::transmute(cap_user_time) }; + cap_user_time as u64 + }); + __bindgen_bitfield_unit.set(4usize, 1u8, { + let cap_user_time_zero: u64 = unsafe { ::core::mem::transmute(cap_user_time_zero) }; + cap_user_time_zero as u64 + }); + __bindgen_bitfield_unit.set(5usize, 1u8, { + let cap_user_time_short: u64 = unsafe { ::core::mem::transmute(cap_user_time_short) }; + cap_user_time_short as u64 + }); + __bindgen_bitfield_unit.set(6usize, 58u8, { + let cap_____res: u64 = unsafe { ::core::mem::transmute(cap_____res) }; + cap_____res as u64 + }); + __bindgen_bitfield_unit + } +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct perf_event_header { + pub type_: __u32, + pub misc: __u16, + pub size: __u16, +} +#[repr(u32)] +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq, ToPrimitive)] +pub enum perf_event_type { + PERF_RECORD_MMAP = 1, + PERF_RECORD_LOST = 2, + PERF_RECORD_COMM = 3, + PERF_RECORD_EXIT = 4, + PERF_RECORD_THROTTLE = 5, + PERF_RECORD_UNTHROTTLE = 6, + PERF_RECORD_FORK = 7, + PERF_RECORD_READ = 8, + PERF_RECORD_SAMPLE = 9, + PERF_RECORD_MMAP2 = 10, + PERF_RECORD_AUX = 11, + PERF_RECORD_ITRACE_START = 12, + PERF_RECORD_LOST_SAMPLES = 13, + PERF_RECORD_SWITCH = 14, + PERF_RECORD_SWITCH_CPU_WIDE = 15, + PERF_RECORD_NAMESPACES = 16, + PERF_RECORD_KSYMBOL = 17, + PERF_RECORD_BPF_EVENT = 18, + PERF_RECORD_CGROUP = 19, + PERF_RECORD_TEXT_POKE = 20, + PERF_RECORD_AUX_OUTPUT_HW_ID = 21, + PERF_RECORD_MAX = 22, +} +pub const TCA_BPF_UNSPEC: _bindgen_ty_152 = 0; +pub const TCA_BPF_ACT: _bindgen_ty_152 = 1; +pub const TCA_BPF_POLICE: _bindgen_ty_152 = 2; +pub const TCA_BPF_CLASSID: _bindgen_ty_152 = 3; +pub const TCA_BPF_OPS_LEN: _bindgen_ty_152 = 4; +pub const TCA_BPF_OPS: _bindgen_ty_152 = 5; +pub const TCA_BPF_FD: _bindgen_ty_152 = 6; +pub const TCA_BPF_NAME: _bindgen_ty_152 = 7; +pub const TCA_BPF_FLAGS: _bindgen_ty_152 = 8; +pub const TCA_BPF_FLAGS_GEN: _bindgen_ty_152 = 9; +pub const TCA_BPF_TAG: _bindgen_ty_152 = 10; +pub const TCA_BPF_ID: _bindgen_ty_152 = 11; +pub const __TCA_BPF_MAX: _bindgen_ty_152 = 12; +pub type _bindgen_ty_152 = ::core::ffi::c_uint; +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct ifinfomsg { + pub ifi_family: ::core::ffi::c_uchar, + pub __ifi_pad: ::core::ffi::c_uchar, + pub ifi_type: ::core::ffi::c_ushort, + pub ifi_index: ::core::ffi::c_int, + pub ifi_flags: ::core::ffi::c_uint, + pub ifi_change: ::core::ffi::c_uint, +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct tcmsg { + pub tcm_family: ::core::ffi::c_uchar, + pub tcm__pad1: ::core::ffi::c_uchar, + pub tcm__pad2: ::core::ffi::c_ushort, + pub tcm_ifindex: ::core::ffi::c_int, + pub tcm_handle: __u32, + pub tcm_parent: __u32, + pub tcm_info: __u32, +} +pub const TCA_UNSPEC: _bindgen_ty_170 = 0; +pub const TCA_KIND: _bindgen_ty_170 = 1; +pub const TCA_OPTIONS: _bindgen_ty_170 = 2; +pub const TCA_STATS: _bindgen_ty_170 = 3; +pub const TCA_XSTATS: _bindgen_ty_170 = 4; +pub const TCA_RATE: _bindgen_ty_170 = 5; +pub const TCA_FCNT: _bindgen_ty_170 = 6; +pub const TCA_STATS2: _bindgen_ty_170 = 7; +pub const TCA_STAB: _bindgen_ty_170 = 8; +pub const TCA_PAD: _bindgen_ty_170 = 9; +pub const TCA_DUMP_INVISIBLE: _bindgen_ty_170 = 10; +pub const TCA_CHAIN: _bindgen_ty_170 = 11; +pub const TCA_HW_OFFLOAD: _bindgen_ty_170 = 12; +pub const TCA_INGRESS_BLOCK: _bindgen_ty_170 = 13; +pub const TCA_EGRESS_BLOCK: _bindgen_ty_170 = 14; +pub const __TCA_MAX: _bindgen_ty_170 = 15; +pub type _bindgen_ty_170 = ::core::ffi::c_uint; +pub const AYA_PERF_EVENT_IOC_ENABLE: ::core::ffi::c_int = 9216; +pub const AYA_PERF_EVENT_IOC_DISABLE: ::core::ffi::c_int = 9217; +pub const AYA_PERF_EVENT_IOC_SET_BPF: ::core::ffi::c_int = 1074013192; diff --git a/kernel/src/include/bindings/mod.rs b/kernel/src/include/bindings/mod.rs index ee9997615..4ab2c21e2 100644 --- a/kernel/src/include/bindings/mod.rs +++ b/kernel/src/include/bindings/mod.rs @@ -1,2 +1,10 @@ -#[allow(clippy::module_inception)] +#![allow( + dead_code, + non_camel_case_types, + non_snake_case, + clippy::all, + missing_docs, + clippy::module_inception +)] pub mod bindings; +pub mod linux_bpf; diff --git a/kernel/src/init/init.rs b/kernel/src/init/init.rs index 953f0974b..653e2f4db 100644 --- a/kernel/src/init/init.rs +++ b/kernel/src/init/init.rs @@ -1,5 +1,3 @@ -use log::warn; - use crate::{ arch::{ init::{early_setup_arch, setup_arch, setup_arch_post}, @@ -30,6 +28,7 @@ use crate::{ clocksource::clocksource_boot_finish, timekeeping::timekeeping_init, timer::timer_init, }, }; +use log::warn; use super::{ boot::{boot_callback_except_early, boot_callbacks}, @@ -89,9 +88,8 @@ fn do_start_kernel() { kthread_init(); setup_arch_post().expect("setup_arch_post failed"); clocksource_boot_finish(); - Futex::init(); - + crate::bpf::init_bpf_system(); #[cfg(all(target_arch = "x86_64", feature = "kvm"))] crate::virt::kvm::kvm_init(); } diff --git a/kernel/src/lib.rs b/kernel/src/lib.rs index aa67c0a7d..5a7e09535 100644 --- a/kernel/src/lib.rs +++ b/kernel/src/lib.rs @@ -21,6 +21,7 @@ #![feature(slice_ptr_get)] #![feature(sync_unsafe_cell)] #![feature(vec_into_raw_parts)] +#![feature(c_variadic)] #![cfg_attr(target_os = "none", no_std)] #![allow(internal_features)] // clippy的配置 @@ -46,6 +47,7 @@ mod arch; mod libs; #[macro_use] mod include; +mod bpf; mod debug; mod driver; // 如果driver依赖了libs，应该在libs后面导出 mod exception; @@ -55,12 +57,12 @@ mod ipc; mod misc; mod mm; mod net; +mod perf; mod process; mod sched; mod smp; mod syscall; mod time; - #[cfg(target_arch = "x86_64")] mod virt; diff --git a/kernel/src/mm/fault.rs b/kernel/src/mm/fault.rs index 457b911f3..e0aa1622a 100644 --- a/kernel/src/mm/fault.rs +++ b/kernel/src/mm/fault.rs @@ -272,16 +272,16 @@ impl PageFaultHandler { /// - VmFaultReason: 页面错误处理信息标志 pub unsafe fn do_fault(pfm: &mut PageFaultMessage) -> VmFaultReason { if !pfm.flags().contains(FaultFlags::FAULT_FLAG_WRITE) { - return Self::do_read_fault(pfm); + Self::do_read_fault(pfm) } else if !pfm .vma() .lock_irqsave() .vm_flags() .contains(VmFlags::VM_SHARED) { - return Self::do_cow_fault(pfm); + Self::do_cow_fault(pfm) } else { - return Self::do_shared_fault(pfm); + Self::do_shared_fault(pfm) } } diff --git a/kernel/src/mm/ucontext.rs b/kernel/src/mm/ucontext.rs index 8892bf2c5..81abd4b0c 100644 --- a/kernel/src/mm/ucontext.rs +++ b/kernel/src/mm/ucontext.rs @@ -377,7 +377,7 @@ impl InnerAddressSpace { PageFrameCount::from_bytes(len).unwrap(), prot_flags, map_flags, - move |page, count, vm_flags, flags, mapper, flusher| { + |page, count, vm_flags, flags, mapper, flusher| { if allocate_at_once { VMA::zeroed( page, @@ -386,7 +386,7 @@ impl InnerAddressSpace { flags, mapper, flusher, - file, + file.clone(), Some(pgoff), ) } else { @@ -394,13 +394,17 @@ impl InnerAddressSpace { VirtRegion::new(page.virt_address(), count.data() * MMArch::PAGE_SIZE), vm_flags, flags, - file, + file.clone(), Some(pgoff), false, ))) } }, )?; + // todo!(impl mmap for other file) + // https://github.com/DragonOS-Community/DragonOS/pull/912#discussion_r1765334272 + let file = file.unwrap(); + let _ = file.inode().mmap(start_vaddr.data(), len, offset); return Ok(start_page); } diff --git a/kernel/src/perf/bpf.rs b/kernel/src/perf/bpf.rs new file mode 100644 index 000000000..84af5436c --- /dev/null +++ b/kernel/src/perf/bpf.rs @@ -0,0 +1,333 @@ +use super::{PerfEventOps, Result}; +use crate::arch::mm::LockedFrameAllocator; +use crate::arch::MMArch; +use crate::filesystem::vfs::file::PageCache; +use crate::filesystem::vfs::{FilePrivateData, FileSystem, IndexNode}; +use crate::include::bindings::linux_bpf::{ + perf_event_header, perf_event_mmap_page, perf_event_type, +}; +use crate::libs::spinlock::{SpinLock, SpinLockGuard}; +use crate::mm::allocator::page_frame::{FrameAllocator, PageFrameCount, PhysPageFrame}; +use crate::mm::page::{page_manager_lock_irqsave, Page}; +use crate::mm::{MemoryManagementArch, PhysAddr}; +use crate::perf::util::{LostSamples, PerfProbeArgs, PerfSample, SampleHeader}; +use alloc::string::String; +use alloc::sync::Arc; +use alloc::vec::Vec; +use core::any::Any; +use core::fmt::Debug; +use system_error::SystemError; +const PAGE_SIZE: usize = MMArch::PAGE_SIZE; +#[derive(Debug)] +pub struct BpfPerfEvent { + _args: PerfProbeArgs, + data: SpinLock, +} + +#[derive(Debug)] +pub struct BpfPerfEventData { + enabled: bool, + mmap_page: RingPage, + page_cache: Arc, + offset: usize, +} + +#[derive(Debug)] +pub struct RingPage { + size: usize, + ptr: usize, + data_region_size: usize, + lost: usize, + phys_addr: PhysAddr, +} + +impl RingPage { + pub fn empty() -> Self { + RingPage { + ptr: 0, + size: 0, + data_region_size: 0, + lost: 0, + phys_addr: PhysAddr::new(0), + } + } + + pub fn new_init(start: usize, len: usize, phys_addr: PhysAddr) -> Self { + Self::init(start as _, len, phys_addr) + } + + fn init(ptr: *mut u8, size: usize, phys_addr: PhysAddr) -> Self { + assert_eq!(size % PAGE_SIZE, 0); + assert!(size / PAGE_SIZE >= 2); + // The first page will be filled with perf_event_mmap_page + unsafe { + let perf_event_mmap_page = &mut *(ptr as *mut perf_event_mmap_page); + perf_event_mmap_page.data_offset = PAGE_SIZE as u64; + perf_event_mmap_page.data_size = (size - PAGE_SIZE) as u64; + // user will read sample or lost record from data_tail + perf_event_mmap_page.data_tail = 0; + // kernel will write sample or lost record from data_head + perf_event_mmap_page.data_head = 0; + // It is a ring buffer. + } + RingPage { + ptr: ptr as usize, + size, + data_region_size: size - PAGE_SIZE, + lost: 0, + phys_addr, + } + } + + fn can_write(&self, data_size: usize, data_tail: usize, data_head: usize) -> bool { + if (data_head + 1) % self.data_region_size == data_tail { + // The buffer is full + return false; + } + let capacity = if data_head >= data_tail { + self.data_region_size - data_head + data_tail + } else { + data_tail - data_head + }; + data_size <= capacity + } + + pub fn write_event(&mut self, data: &[u8]) -> Result<()> { + let data_tail = unsafe { &mut (*(self.ptr as *mut perf_event_mmap_page)).data_tail }; + let data_head = unsafe { &mut (*(self.ptr as *mut perf_event_mmap_page)).data_head }; + // data_tail..data_head is the region that can be written + // check if there is enough space to write the event + let sample_size = PerfSample::calculate_size(data.len()); + + let can_write_sample = + self.can_write(sample_size, *data_tail as usize, *data_head as usize); + // log::error!( + // "can_write_sample: {}, data_tail: {}, data_head: {}, data.len(): {}, region_size: {}", + // can_write_sample, + // *data_tail, + // *data_head, + // data.len(), + // self.data_region_size + // ); + if !can_write_sample { + //we need record it to the lost record + self.lost += 1; + // log::error!( + // "Lost record: {}, data_tail: {}, data_head: {}", + // self.lost, + // *data_tail, + // *data_head + // ); + Ok(()) + } else { + // we can write the sample to the page + // If the lost record is not zero, we need to write the lost record first. + let can_write_lost_record = self.can_write( + size_of::(), + *data_tail as usize, + *data_head as usize, + ); + if self.lost > 0 && can_write_lost_record { + let new_data_head = self.write_lost(*data_head as usize)?; + *data_head = new_data_head as u64; + // log::info!( + // "Write lost record: {}, data_tail: {}, new_data_head: {}", + // self.lost, + // *data_tail, + // *data_head + // ); + self.lost = 0; + self.write_event(data) + } else { + let new_data_head = self.write_sample(data, *data_head as usize)?; + *data_head = new_data_head as u64; + // log::info!( + // "Write sample record, data_tail: {}, new_data_head: {}", + // *data_tail, + // *data_head + // ); + Ok(()) + } + } + } + + /// Write any data to the page. + /// + /// Return the new data_head + fn write_any(&mut self, data: &[u8], data_head: usize) -> Result { + let data_region_len = self.data_region_size; + let data_region = self.as_mut_slice()[PAGE_SIZE..].as_mut(); + let data_len = data.len(); + let end = (data_head + data_len) % data_region_len; + let start = data_head; + if start < end { + data_region[start..end].copy_from_slice(data); + } else { + let first_len = data_region_len - start; + data_region[start..start + first_len].copy_from_slice(&data[..first_len]); + data_region[0..end].copy_from_slice(&data[first_len..]); + } + Ok(end) + } + + /// Write a sample to the page. + fn write_sample(&mut self, data: &[u8], data_head: usize) -> Result { + let perf_sample = PerfSample { + s_hdr: SampleHeader { + header: perf_event_header { + type_: perf_event_type::PERF_RECORD_SAMPLE as u32, + misc: 0, + size: size_of::() as u16 + data.len() as u16, + }, + size: data.len() as u32, + }, + value: data, + }; + let new_head = self.write_any(perf_sample.s_hdr.as_bytes(), data_head)?; + self.write_any(perf_sample.value, new_head) + } + + /// Write a lost record to the page. + /// + /// Return the new data_head + fn write_lost(&mut self, data_head: usize) -> Result { + let lost = LostSamples { + header: perf_event_header { + type_: perf_event_type::PERF_RECORD_LOST as u32, + misc: 0, + size: size_of::() as u16, + }, + id: 0, + count: self.lost as u64, + }; + self.write_any(lost.as_bytes(), data_head) + } + + pub fn readable(&self) -> bool { + let data_tail = unsafe { &(*(self.ptr as *mut perf_event_mmap_page)).data_tail }; + let data_head = unsafe { &(*(self.ptr as *mut perf_event_mmap_page)).data_head }; + data_tail != data_head + } + pub fn as_slice(&self) -> &[u8] { + unsafe { core::slice::from_raw_parts(self.ptr as *const u8, self.size) } + } + pub fn as_mut_slice(&mut self) -> &mut [u8] { + unsafe { core::slice::from_raw_parts_mut(self.ptr as *mut u8, self.size) } + } +} + +impl BpfPerfEvent { + pub fn new(args: PerfProbeArgs) -> Self { + BpfPerfEvent { + _args: args, + data: SpinLock::new(BpfPerfEventData { + enabled: false, + mmap_page: RingPage::empty(), + page_cache: PageCache::new(None), + offset: 0, + }), + } + } + pub fn do_mmap(&self, _start: usize, len: usize, offset: usize) -> Result<()> { + let mut data = self.data.lock(); + // alloc page frame + let (phy_addr, page_count) = + unsafe { LockedFrameAllocator.allocate(PageFrameCount::new(len / PAGE_SIZE)) } + .ok_or(SystemError::ENOSPC)?; + let mut page_manager_guard = page_manager_lock_irqsave(); + let mut cur_phys = PhysPageFrame::new(phy_addr); + for i in 0..page_count.data() { + let page = Arc::new(Page::new(true, cur_phys.phys_address())); + let paddr = cur_phys.phys_address(); + page_manager_guard.insert(paddr, &page); + data.page_cache.add_page(i, &page); + cur_phys = cur_phys.next(); + } + let virt_addr = unsafe { MMArch::phys_2_virt(phy_addr) }.ok_or(SystemError::EFAULT)?; + // create mmap page + let mmap_page = RingPage::new_init(virt_addr.data(), len, phy_addr); + data.mmap_page = mmap_page; + data.offset = offset; + Ok(()) + } + + pub fn write_event(&self, data: &[u8]) -> Result<()> { + let mut inner_data = self.data.lock(); + inner_data.mmap_page.write_event(data)?; + Ok(()) + } +} + +impl Drop for BpfPerfEvent { + fn drop(&mut self) { + let mut page_manager_guard = page_manager_lock_irqsave(); + let data = self.data.lock(); + let phy_addr = data.mmap_page.phys_addr; + let len = data.mmap_page.size; + let page_count = PageFrameCount::new(len / PAGE_SIZE); + let mut cur_phys = PhysPageFrame::new(phy_addr); + for _ in 0..page_count.data() { + page_manager_guard.remove_page(&cur_phys.phys_address()); + cur_phys = cur_phys.next(); + } + } +} + +impl IndexNode for BpfPerfEvent { + fn mmap(&self, start: usize, len: usize, offset: usize) -> Result<()> { + self.do_mmap(start, len, offset) + } + + fn read_at( + &self, + _offset: usize, + _len: usize, + _buf: &mut [u8], + _data: SpinLockGuard, + ) -> Result { + panic!("PerfEventInode does not support read") + } + + fn write_at( + &self, + _offset: usize, + _len: usize, + _buf: &[u8], + _data: SpinLockGuard, + ) -> Result { + panic!("PerfEventInode does not support write") + } + + fn fs(&self) -> Arc { + panic!("PerfEventInode does not have a filesystem") + } + + fn as_any_ref(&self) -> &dyn Any { + self + } + fn list(&self) -> Result> { + Err(SystemError::ENOSYS) + } + + fn page_cache(&self) -> Option> { + Some(self.data.lock().page_cache.clone()) + } +} + +impl PerfEventOps for BpfPerfEvent { + fn enable(&self) -> Result<()> { + self.data.lock().enabled = true; + Ok(()) + } + fn disable(&self) -> Result<()> { + self.data.lock().enabled = false; + Ok(()) + } + fn readable(&self) -> bool { + self.data.lock().mmap_page.readable() + } +} + +pub fn perf_event_open_bpf(args: PerfProbeArgs) -> BpfPerfEvent { + BpfPerfEvent::new(args) +} diff --git a/kernel/src/perf/kprobe.rs b/kernel/src/perf/kprobe.rs new file mode 100644 index 000000000..54ac829bc --- /dev/null +++ b/kernel/src/perf/kprobe.rs @@ -0,0 +1,159 @@ +use super::Result; +use crate::arch::interrupt::TrapFrame; +use crate::arch::kprobe::KProbeContext; +use crate::bpf::helper::BPF_HELPER_FUN_SET; +use crate::bpf::prog::BpfProg; +use crate::debug::kprobe::args::KprobeInfo; +use crate::debug::kprobe::{register_kprobe, unregister_kprobe, LockKprobe}; +use crate::filesystem::vfs::file::{File, PageCache}; +use crate::filesystem::vfs::{FilePrivateData, FileSystem, IndexNode}; +use crate::libs::casting::DowncastArc; +use crate::libs::spinlock::SpinLockGuard; +use crate::perf::util::PerfProbeArgs; +use crate::perf::PerfEventOps; +use alloc::boxed::Box; +use alloc::string::String; +use alloc::sync::Arc; +use alloc::vec::Vec; +use core::any::Any; +use core::fmt::Debug; +use kprobe::{CallBackFunc, ProbeArgs}; +use rbpf::EbpfVmRawOwned; +use system_error::SystemError; +#[derive(Debug)] +pub struct KprobePerfEvent { + _args: PerfProbeArgs, + kprobe: LockKprobe, +} + +impl Drop for KprobePerfEvent { + fn drop(&mut self) { + unregister_kprobe(self.kprobe.clone()); + } +} + +impl KprobePerfEvent { + pub fn do_set_bpf_prog(&self, prog_file: Arc) -> Result<()> { + let file = prog_file + .inode() + .downcast_arc::() + .ok_or(SystemError::EINVAL)?; + let prog_slice = file.insns(); + let mut vm = + EbpfVmRawOwned::new(Some(prog_slice.to_vec())).map_err(|_| SystemError::EINVAL)?; + vm.register_helper_set(BPF_HELPER_FUN_SET.get()) + .map_err(|_| SystemError::EINVAL)?; + // create a callback to execute the ebpf prog + let callback = Box::new(KprobePerfCallBack::new(file, vm)); + // update callback for kprobe + self.kprobe.write().update_event_callback(callback); + Ok(()) + } +} + +pub struct KprobePerfCallBack { + _bpf_prog_file: Arc, + vm: EbpfVmRawOwned, +} + +impl KprobePerfCallBack { + fn new(bpf_prog_file: Arc, vm: EbpfVmRawOwned) -> Self { + Self { + _bpf_prog_file: bpf_prog_file, + vm, + } + } +} + +impl CallBackFunc for KprobePerfCallBack { + fn call(&self, trap_frame: &dyn ProbeArgs) { + let trap_frame = trap_frame.as_any().downcast_ref::().unwrap(); + let pt_regs = KProbeContext::from(trap_frame); + let probe_context = unsafe { + core::slice::from_raw_parts_mut( + &pt_regs as *const KProbeContext as *mut u8, + size_of::(), + ) + }; + let _res = self + .vm + .execute_program(probe_context) + .map_err(|_| SystemError::EINVAL); + } +} + +impl IndexNode for KprobePerfEvent { + fn read_at( + &self, + _offset: usize, + _len: usize, + _buf: &mut [u8], + _data: SpinLockGuard, + ) -> Result { + panic!("read_at not implemented for PerfEvent"); + } + + fn write_at( + &self, + _offset: usize, + _len: usize, + _buf: &[u8], + _data: SpinLockGuard, + ) -> Result { + panic!("write_at not implemented for PerfEvent"); + } + + fn fs(&self) -> Arc { + panic!("fs not implemented for PerfEvent"); + } + + fn as_any_ref(&self) -> &dyn Any { + self + } + + fn list(&self) -> Result> { + Err(SystemError::ENOSYS) + } + + fn page_cache(&self) -> Option> { + None + } +} + +impl PerfEventOps for KprobePerfEvent { + fn set_bpf_prog(&self, bpf_prog: Arc) -> Result<()> { + self.do_set_bpf_prog(bpf_prog) + } + fn enable(&self) -> Result<()> { + self.kprobe.write().enable(); + Ok(()) + } + fn disable(&self) -> Result<()> { + self.kprobe.write().disable(); + Ok(()) + } + + fn readable(&self) -> bool { + true + } +} + +pub fn perf_event_open_kprobe(args: PerfProbeArgs) -> KprobePerfEvent { + let symbol = args.name.clone(); + log::info!("create kprobe for symbol: {symbol}"); + let kprobe_info = KprobeInfo { + pre_handler: |_| {}, + post_handler: |_| {}, + fault_handler: None, + event_callback: None, + symbol: Some(symbol), + addr: None, + offset: 0, + enable: false, + }; + let kprobe = register_kprobe(kprobe_info).expect("create kprobe failed"); + KprobePerfEvent { + _args: args, + kprobe, + } +} diff --git a/kernel/src/perf/mod.rs b/kernel/src/perf/mod.rs new file mode 100644 index 000000000..0b6de2adb --- /dev/null +++ b/kernel/src/perf/mod.rs @@ -0,0 +1,337 @@ +mod bpf; +mod kprobe; +mod util; + +use crate::filesystem::vfs::file::{File, FileMode, PageCache}; +use crate::filesystem::vfs::syscall::ModeType; +use crate::filesystem::vfs::{ + FilePrivateData, FileSystem, FileType, FsInfo, IndexNode, Metadata, SuperBlock, +}; +use crate::include::bindings::linux_bpf::{ + perf_event_attr, perf_event_sample_format, perf_sw_ids, perf_type_id, +}; +use crate::libs::casting::DowncastArc; +use crate::libs::spinlock::{SpinLock, SpinLockGuard}; +use crate::mm::fault::{PageFaultHandler, PageFaultMessage}; +use crate::mm::VmFaultReason; +use crate::net::event_poll::{EPollEventType, EPollItem, EventPoll, KernelIoctlData}; +use crate::perf::bpf::BpfPerfEvent; +use crate::perf::util::{PerfEventIoc, PerfEventOpenFlags, PerfProbeArgs}; +use crate::process::ProcessManager; +use crate::syscall::user_access::UserBufferReader; +use crate::syscall::Syscall; +use alloc::boxed::Box; +use alloc::collections::LinkedList; +use alloc::string::String; +use alloc::sync::{Arc, Weak}; +use alloc::vec::Vec; +use core::any::Any; +use core::ffi::c_void; +use core::fmt::Debug; +use core::ops::Deref; +use intertrait::{CastFrom, CastFromSync}; +use log::info; +use num_traits::FromPrimitive; +use system_error::SystemError; + +type Result = core::result::Result; + +pub trait PerfEventOps: Send + Sync + Debug + CastFromSync + CastFrom + IndexNode { + /// Set the bpf program for the perf event + fn set_bpf_prog(&self, _bpf_prog: Arc) -> Result<()> { + Err(SystemError::ENOSYS) + } + /// Enable the perf event + fn enable(&self) -> Result<()> { + Err(SystemError::ENOSYS) + } + /// Disable the perf event + fn disable(&self) -> Result<()> { + Err(SystemError::ENOSYS) + } + /// Whether the perf event is readable + fn readable(&self) -> bool; +} + +#[derive(Debug)] +pub struct PerfEventInode { + event: Box, + epitems: SpinLock>>, +} + +impl PerfEventInode { + pub fn new(event: Box) -> Self { + Self { + event, + epitems: SpinLock::new(LinkedList::new()), + } + } + pub fn remove_epoll( + &self, + epoll: &Weak>, + ) -> core::result::Result<(), SystemError> { + let is_remove = !self + .epitems + .lock_irqsave() + .extract_if(|x| x.epoll().ptr_eq(epoll)) + .collect::>() + .is_empty(); + if is_remove { + return Ok(()); + } + Err(SystemError::ENOENT) + } + fn do_poll(&self) -> Result { + let mut events = EPollEventType::empty(); + if self.event.readable() { + events |= EPollEventType::EPOLLIN | EPollEventType::EPOLLRDNORM; + } + return Ok(events.bits() as usize); + } + fn epoll_callback(&self) -> Result<()> { + let pollflag = EPollEventType::from_bits_truncate(self.do_poll()? as u32); + // 唤醒epoll中等待的进程 + EventPoll::wakeup_epoll(&self.epitems, Some(pollflag)) + } +} + +impl Deref for PerfEventInode { + type Target = Box; + + fn deref(&self) -> &Self::Target { + &self.event + } +} + +impl IndexNode for PerfEventInode { + fn mmap(&self, start: usize, len: usize, offset: usize) -> Result<()> { + self.event.mmap(start, len, offset) + } + fn open(&self, _data: SpinLockGuard, _mode: &FileMode) -> Result<()> { + Ok(()) + } + fn close(&self, _data: SpinLockGuard) -> Result<()> { + Ok(()) + } + fn read_at( + &self, + _offset: usize, + _len: usize, + _buf: &mut [u8], + _data: SpinLockGuard, + ) -> Result { + panic!("read_at not implemented for PerfEvent"); + } + + fn write_at( + &self, + _offset: usize, + _len: usize, + _buf: &[u8], + _data: SpinLockGuard, + ) -> Result { + panic!("write_at not implemented for PerfEvent"); + } + + fn poll(&self, _private_data: &FilePrivateData) -> Result { + self.do_poll() + } + + fn metadata(&self) -> Result { + let meta = Metadata { + mode: ModeType::from_bits_truncate(0o755), + file_type: FileType::File, + ..Default::default() + }; + Ok(meta) + } + + fn resize(&self, _len: usize) -> Result<()> { + Ok(()) + } + + fn ioctl(&self, cmd: u32, data: usize, _private_data: &FilePrivateData) -> Result { + let req = PerfEventIoc::from_u32(cmd).ok_or(SystemError::EINVAL)?; + info!("perf_event_ioctl: request: {:?}, arg: {}", req, data); + match req { + PerfEventIoc::Enable => { + self.event.enable()?; + Ok(0) + } + PerfEventIoc::Disable => { + self.event.disable()?; + Ok(0) + } + PerfEventIoc::SetBpf => { + info!("perf_event_ioctl: PERF_EVENT_IOC_SET_BPF, arg: {}", data); + let bpf_prog_fd = data; + let fd_table = ProcessManager::current_pcb().fd_table(); + let file = fd_table + .read() + .get_file_by_fd(bpf_prog_fd as _) + .ok_or(SystemError::EBADF)?; + self.event.set_bpf_prog(file)?; + Ok(0) + } + } + } + + fn kernel_ioctl( + &self, + arg: Arc, + _data: &FilePrivateData, + ) -> core::result::Result { + let epitem = arg + .arc_any() + .downcast::() + .map_err(|_| SystemError::EFAULT)?; + self.epitems.lock().push_back(epitem); + Ok(0) + } + + fn fs(&self) -> Arc { + // panic!("PerfEvent does not have a filesystem") + Arc::new(PerfFakeFs) + } + fn as_any_ref(&self) -> &dyn Any { + self + } + fn list(&self) -> Result> { + Err(SystemError::ENOSYS) + } + fn page_cache(&self) -> Option> { + self.event.page_cache() + } +} + +#[derive(Debug)] +struct PerfFakeFs; + +impl FileSystem for PerfFakeFs { + fn root_inode(&self) -> Arc { + panic!("PerfFakeFs does not have a root inode") + } + + fn info(&self) -> FsInfo { + panic!("PerfFakeFs does not have a filesystem info") + } + + fn as_any_ref(&self) -> &dyn Any { + self + } + + fn name(&self) -> &str { + "perf" + } + + fn super_block(&self) -> SuperBlock { + panic!("PerfFakeFs does not have a super block") + } + unsafe fn fault(&self, pfm: &mut PageFaultMessage) -> VmFaultReason { + let res = PageFaultHandler::filemap_fault(pfm); + res + } + unsafe fn map_pages( + &self, + pfm: &mut PageFaultMessage, + start_pgoff: usize, + end_pgoff: usize, + ) -> VmFaultReason { + PageFaultHandler::filemap_map_pages(pfm, start_pgoff, end_pgoff) + } +} + +impl Syscall { + pub fn sys_perf_event_open( + attr: *const u8, + pid: i32, + cpu: i32, + group_fd: i32, + flags: u32, + ) -> Result { + let buf = UserBufferReader::new( + attr as *const perf_event_attr, + size_of::(), + true, + )?; + let attr = buf.read_one_from_user(0)?; + perf_event_open(attr, pid, cpu, group_fd, flags) + } +} + +pub fn perf_event_open( + attr: &perf_event_attr, + pid: i32, + cpu: i32, + group_fd: i32, + flags: u32, +) -> Result { + let args = PerfProbeArgs::try_from(attr, pid, cpu, group_fd, flags)?; + log::info!("perf_event_process: {:#?}", args); + let file_mode = if args + .flags + .contains(PerfEventOpenFlags::PERF_FLAG_FD_CLOEXEC) + { + FileMode::O_RDWR | FileMode::O_CLOEXEC + } else { + FileMode::O_RDWR + }; + + let event: Box = match args.type_ { + // Kprobe + // See /sys/bus/event_source/devices/kprobe/type + perf_type_id::PERF_TYPE_MAX => { + let kprobe_event = kprobe::perf_event_open_kprobe(args); + Box::new(kprobe_event) + } + perf_type_id::PERF_TYPE_SOFTWARE => { + // For bpf prog output + assert_eq!(args.config, perf_sw_ids::PERF_COUNT_SW_BPF_OUTPUT); + assert_eq!( + args.sample_type, + Some(perf_event_sample_format::PERF_SAMPLE_RAW) + ); + let bpf_event = bpf::perf_event_open_bpf(args); + Box::new(bpf_event) + } + _ => { + unimplemented!("perf_event_process: unknown type: {:?}", args); + } + }; + + let page_cache = event.page_cache(); + let perf_event = Arc::new(PerfEventInode::new(event)); + if let Some(cache) = page_cache { + cache.set_inode(Arc::downgrade(&(perf_event.clone() as _)))?; + } + let file = File::new(perf_event, file_mode)?; + let fd_table = ProcessManager::current_pcb().fd_table(); + let fd = fd_table.write().alloc_fd(file, None).map(|x| x as usize)?; + Ok(fd) +} + +pub fn perf_event_output(_ctx: *mut c_void, fd: usize, _flags: u32, data: &[u8]) -> Result<()> { + let file = get_perf_event_file(fd)?; + let bpf_event_file = file.deref().deref(); + let bpf_event_file = bpf_event_file + .deref() + .ref_any() + .downcast_ref::() + .ok_or(SystemError::EINVAL)?; + bpf_event_file.write_event(data)?; + file.epoll_callback()?; + Ok(()) +} + +fn get_perf_event_file(fd: usize) -> Result> { + let fd_table = ProcessManager::current_pcb().fd_table(); + let file = fd_table + .read() + .get_file_by_fd(fd as _) + .ok_or(SystemError::EBADF)?; + let event = file + .inode() + .downcast_arc::() + .ok_or(SystemError::EINVAL)?; + Ok(event) +} diff --git a/kernel/src/perf/util.rs b/kernel/src/perf/util.rs new file mode 100644 index 000000000..b1cc43eb4 --- /dev/null +++ b/kernel/src/perf/util.rs @@ -0,0 +1,123 @@ +use crate::include::bindings::linux_bpf::{ + perf_event_attr, perf_event_header, perf_event_sample_format, perf_sw_ids, perf_type_id, +}; +use crate::syscall::user_access::check_and_clone_cstr; +use alloc::string::String; +use num_traits::FromPrimitive; +use system_error::SystemError; + +bitflags! { + pub struct PerfEventOpenFlags: u32 { + const PERF_FLAG_FD_NO_GROUP = 1; + const PERF_FLAG_FD_OUTPUT = 2; + const PERF_FLAG_PID_CGROUP = 4; + const PERF_FLAG_FD_CLOEXEC = 8; + } +} + +/// The `PerfEventIoc` enum is used to define the ioctl commands for perf events. +/// +/// See https://elixir.bootlin.com/linux/v6.1/source/include/uapi/linux/perf_event.h#L544 +#[repr(u32)] +#[derive(Debug, Copy, Clone, FromPrimitive)] +pub enum PerfEventIoc { + /// Equivalent to [crate::include::bindings::linux_bpf::AYA_PERF_EVENT_IOC_ENABLE]. + Enable = 9216, + /// Equivalent to [crate::include::bindings::linux_bpf::AYA_PERF_EVENT_IOC_DISABLE]. + Disable = 9217, + /// Equivalent to [crate::include::bindings::linux_bpf::AYA_PERF_EVENT_IOC_SET_BPF]. + SetBpf = 1074013192, +} + +#[derive(Debug, Clone)] +#[allow(unused)] +/// `perf_event_open` syscall arguments. +pub struct PerfProbeArgs { + pub config: perf_sw_ids, + pub name: String, + pub offset: u64, + pub size: u32, + pub type_: perf_type_id, + pub pid: i32, + pub cpu: i32, + pub group_fd: i32, + pub flags: PerfEventOpenFlags, + pub sample_type: Option, +} + +impl PerfProbeArgs { + pub fn try_from( + attr: &perf_event_attr, + pid: i32, + cpu: i32, + group_fd: i32, + flags: u32, + ) -> Result { + let ty = perf_type_id::from_u32(attr.type_).ok_or(SystemError::EINVAL)?; + let config = perf_sw_ids::from_u32(attr.config as u32).ok_or(SystemError::EINVAL)?; + let name = if ty == perf_type_id::PERF_TYPE_MAX { + let name_ptr = unsafe { attr.__bindgen_anon_3.config1 } as *const u8; + let name = check_and_clone_cstr(name_ptr, None)?; + name.into_string().map_err(|_| SystemError::EINVAL)? + } else { + String::new() + }; + let sample_ty = perf_event_sample_format::from_u32(attr.sample_type as u32); + let args = PerfProbeArgs { + config, + name, + offset: unsafe { attr.__bindgen_anon_4.config2 }, + size: attr.size, + type_: ty, + pid, + cpu, + group_fd, + flags: PerfEventOpenFlags::from_bits_truncate(flags), + sample_type: sample_ty, + }; + Ok(args) + } +} + +/// The event type in our particular use case will be `PERF_RECORD_SAMPLE` or `PERF_RECORD_LOST`. +/// `PERF_RECORD_SAMPLE` indicating that there is an actual sample after this header. +/// And `PERF_RECORD_LOST` indicating that there is a record lost header following the perf event header. +#[repr(C)] +#[derive(Debug)] +pub struct LostSamples { + pub header: perf_event_header, + pub id: u64, + pub count: u64, +} + +impl LostSamples { + pub fn as_bytes(&self) -> &[u8] { + unsafe { core::slice::from_raw_parts(self as *const Self as *const u8, size_of::()) } + } +} + +#[repr(C)] +#[derive(Debug)] +pub struct SampleHeader { + pub header: perf_event_header, + pub size: u32, +} + +impl SampleHeader { + pub fn as_bytes(&self) -> &[u8] { + unsafe { core::slice::from_raw_parts(self as *const Self as *const u8, size_of::()) } + } +} + +#[repr(C)] +#[derive(Debug)] +pub struct PerfSample<'a> { + pub s_hdr: SampleHeader, + pub value: &'a [u8], +} + +impl<'a> PerfSample<'a> { + pub fn calculate_size(value_size: usize) -> usize { + size_of::() + value_size + } +} diff --git a/kernel/src/smp/cpu/mod.rs b/kernel/src/smp/cpu/mod.rs index 186184ef5..d3b36002d 100644 --- a/kernel/src/smp/cpu/mod.rs +++ b/kernel/src/smp/cpu/mod.rs @@ -134,7 +134,6 @@ impl SmpCpuManager { &self.possible_cpus } - #[allow(dead_code)] pub fn possible_cpus_count(&self) -> u32 { self.possible_cnt.load(core::sync::atomic::Ordering::SeqCst) } diff --git a/kernel/src/syscall/mod.rs b/kernel/src/syscall/mod.rs index c61c2db57..6e7e9f7be 100644 --- a/kernel/src/syscall/mod.rs +++ b/kernel/src/syscall/mod.rs @@ -1159,6 +1159,20 @@ impl Syscall { let flags = args[1] as u32; Self::sys_eventfd(initval, flags) } + SYS_BPF => { + let cmd = args[0] as u32; + let attr = args[1] as *mut u8; + let size = args[2] as u32; + Self::sys_bpf(cmd, attr, size) + } + SYS_PERF_EVENT_OPEN => { + let attr = args[0] as *const u8; + let pid = args[1] as i32; + let cpu = args[2] as i32; + let group_fd = args[3] as i32; + let flags = args[4] as u32; + Self::sys_perf_event_open(attr, pid, cpu, group_fd, flags) + } _ => panic!("Unsupported syscall ID: {}", syscall_num), }; diff --git a/kernel/src/time/syscall.rs b/kernel/src/time/syscall.rs index 76f9349eb..ec8c0a155 100644 --- a/kernel/src/time/syscall.rs +++ b/kernel/src/time/syscall.rs @@ -2,8 +2,6 @@ use core::{ ffi::{c_int, c_longlong}, time::Duration, }; - -use log::warn; use num_traits::FromPrimitive; use system_error::SystemError; @@ -139,7 +137,7 @@ impl Syscall { pub fn clock_gettime(clock_id: c_int, tp: *mut PosixTimeSpec) -> Result { let clock_id = PosixClockID::try_from(clock_id)?; if clock_id != PosixClockID::Realtime { - warn!("clock_gettime: currently only support Realtime clock, but got {:?}. Defaultly return realtime!!!\n", clock_id); + // warn!("clock_gettime: currently only support Realtime clock, but got {:?}. Defaultly return realtime!!!\n", clock_id); } if tp.is_null() { return Err(SystemError::EFAULT); diff --git a/user/apps/test_ebpf/.gitignore b/user/apps/test_ebpf/.gitignore new file mode 100644 index 000000000..1ac354611 --- /dev/null +++ b/user/apps/test_ebpf/.gitignore @@ -0,0 +1,3 @@ +/target +Cargo.lock +/install/ \ No newline at end of file diff --git a/user/apps/test_ebpf/Cargo.toml b/user/apps/test_ebpf/Cargo.toml new file mode 100644 index 000000000..ab4a3b7f9 --- /dev/null +++ b/user/apps/test_ebpf/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "test_ebpf" +version = "0.1.0" +edition = "2021" + +[dependencies] +aya = { git = "https://git.mirrors.dragonos.org.cn/DragonOS-Community/tiny-aya.git", rev = "0689f13" } +aya-log = { git = "https://git.mirrors.dragonos.org.cn/DragonOS-Community/tiny-aya.git", rev = "0689f13" } + +log = "0.4.22" +env_logger = "0.11.5" +tokio = { version = "1.25", features = ["macros", "rt", "rt-multi-thread", "net", "signal", "time"] } + +[profile.release] +lto = true +strip = true diff --git a/user/apps/test_ebpf/Makefile b/user/apps/test_ebpf/Makefile new file mode 100644 index 000000000..f2770a320 --- /dev/null +++ b/user/apps/test_ebpf/Makefile @@ -0,0 +1,61 @@ +TOOLCHAIN="+nightly-2024-07-23-x86_64-unknown-linux-gnu" +RUSTFLAGS+="" + +ifdef DADK_CURRENT_BUILD_DIR +# 如果是在dadk中编译，那么安装到dadk的安装目录中 + INSTALL_DIR = $(DADK_CURRENT_BUILD_DIR) +else +# 如果是在本地编译，那么安装到当前目录下的install目录中 + INSTALL_DIR = ./install +endif + +ifeq ($(ARCH), x86_64) + export RUST_TARGET=x86_64-unknown-linux-musl +else ifeq ($(ARCH), riscv64) + export RUST_TARGET=riscv64gc-unknown-linux-gnu +else +# 默认为x86_86，用于本地编译 + export RUST_TARGET=x86_64-unknown-linux-musl +endif + +run: + RUSTFLAGS=$(RUSTFLAGS) cargo $(TOOLCHAIN) run --target $(RUST_TARGET) + +build:build-ebpf + RUSTFLAGS=$(RUSTFLAGS) cargo $(TOOLCHAIN) build --target $(RUST_TARGET) + +clean:clean-ebpf + RUSTFLAGS=$(RUSTFLAGS) cargo $(TOOLCHAIN) clean --target $(RUST_TARGET) + +test: + RUSTFLAGS=$(RUSTFLAGS) cargo $(TOOLCHAIN) test --target $(RUST_TARGET) + +doc: + RUSTFLAGS=$(RUSTFLAGS) cargo $(TOOLCHAIN) doc --target $(RUST_TARGET) + +fmt: + RUSTFLAGS=$(RUSTFLAGS) cargo $(TOOLCHAIN) fmt + +fmt-check: + RUSTFLAGS=$(RUSTFLAGS) cargo $(TOOLCHAIN) fmt --check + +run-release: + RUSTFLAGS=$(RUSTFLAGS) cargo $(TOOLCHAIN) run --target $(RUST_TARGET) --release + +build-release:build-ebpf + RUSTFLAGS=$(RUSTFLAGS) cargo $(TOOLCHAIN) build --target $(RUST_TARGET) --release + +clean-release:clean-ebpf + RUSTFLAGS=$(RUSTFLAGS) cargo $(TOOLCHAIN) clean --target $(RUST_TARGET) --release + +test-release: + RUSTFLAGS=$(RUSTFLAGS) cargo $(TOOLCHAIN) test --target $(RUST_TARGET) --release + +build-ebpf: + cd ./syscall_ebpf && RUST_LOG=debug cargo xtask build --release +clean-ebpf: + cd ./syscall_ebpf && cargo clean + +.PHONY: install +install:build-ebpf + RUSTFLAGS=$(RUSTFLAGS) cargo $(TOOLCHAIN) install --target $(RUST_TARGET) --path . --no-track --root $(INSTALL_DIR) --force diff --git a/user/apps/test_ebpf/src/main.rs b/user/apps/test_ebpf/src/main.rs new file mode 100644 index 000000000..1909aeeb9 --- /dev/null +++ b/user/apps/test_ebpf/src/main.rs @@ -0,0 +1,60 @@ +use aya::maps::HashMap; +use aya::programs::KProbe; +use aya::{include_bytes_aligned, Ebpf}; +use aya_log::EbpfLogger; +use log::{info, warn}; +use std::error::Error; +use tokio::task::yield_now; +use tokio::{signal, time}; + +#[tokio::main(flavor = "current_thread")] +async fn main() -> Result<(), Box> { + env_logger::builder() + .filter_level(log::LevelFilter::Warn) + .format_timestamp(None) + .init(); + + let mut bpf = Ebpf::load(include_bytes_aligned!( + "../syscall_ebpf/target/bpfel-unknown-none/release/syscall_ebpf" + ))?; + + // create a async task to read the log + if let Err(e) = EbpfLogger::init(&mut bpf) { + // This can happen if you remove all log statements from your eBPF program. + warn!("failed to initialize eBPF logger: {}", e); + } + + let program: &mut KProbe = bpf.program_mut("syscall_ebpf").unwrap().try_into()?; + program.load()?; + program.attach("dragonos_kernel::syscall::Syscall::handle", 0)?; + + info!("attacch the kprobe to dragonos_kernel::syscall::Syscall::handle"); + + // print the value of the blocklist per 5 seconds + tokio::spawn(async move { + let blocklist: HashMap<_, u32, u32> = + HashMap::try_from(bpf.map("SYSCALL_LIST").unwrap()).unwrap(); + let mut now = time::Instant::now(); + loop { + let new_now = time::Instant::now(); + let duration = new_now.duration_since(now); + if duration.as_secs() >= 5 { + println!("------------SYSCALL_LIST----------------"); + let iter = blocklist.iter(); + for item in iter { + if let Ok((key, value)) = item { + println!("syscall: {:?}, count: {:?}", key, value); + } + } + println!("----------------------------------------"); + now = new_now; + } + yield_now().await; + } + }); + + info!("Waiting for Ctrl-C..."); + signal::ctrl_c().await?; + info!("Exiting..."); + Ok(()) +} diff --git a/user/apps/test_ebpf/syscall_ebpf/.cargo/config.toml b/user/apps/test_ebpf/syscall_ebpf/.cargo/config.toml new file mode 100644 index 000000000..35049cbcb --- /dev/null +++ b/user/apps/test_ebpf/syscall_ebpf/.cargo/config.toml @@ -0,0 +1,2 @@ +[alias] +xtask = "run --package xtask --" diff --git a/user/apps/test_ebpf/syscall_ebpf/.gitignore b/user/apps/test_ebpf/syscall_ebpf/.gitignore new file mode 100644 index 000000000..9db7029fd --- /dev/null +++ b/user/apps/test_ebpf/syscall_ebpf/.gitignore @@ -0,0 +1,9 @@ +### https://raw.github.com/github/gitignore/master/Rust.gitignore + +# Generated by Cargo +# will have compiled files and executables +debug/ +target/ + +# These are backup files generated by rustfmt +**/*.rs.bk diff --git a/user/apps/test_ebpf/syscall_ebpf/.vscode/settings.json b/user/apps/test_ebpf/syscall_ebpf/.vscode/settings.json new file mode 100644 index 000000000..0c82ac973 --- /dev/null +++ b/user/apps/test_ebpf/syscall_ebpf/.vscode/settings.json @@ -0,0 +1,3 @@ +{ + "rust-analyzer.linkedProjects": ["Cargo.toml", "syscall_ebpf-ebpf/Cargo.toml"] +} diff --git a/user/apps/test_ebpf/syscall_ebpf/Cargo.toml b/user/apps/test_ebpf/syscall_ebpf/Cargo.toml new file mode 100644 index 000000000..6eb4e6322 --- /dev/null +++ b/user/apps/test_ebpf/syscall_ebpf/Cargo.toml @@ -0,0 +1,3 @@ +[workspace] +resolver = "2" +members = ["xtask", "syscall_ebpf-common"] diff --git a/user/apps/test_ebpf/syscall_ebpf/README.md b/user/apps/test_ebpf/syscall_ebpf/README.md new file mode 100644 index 000000000..fe5ed32d3 --- /dev/null +++ b/user/apps/test_ebpf/syscall_ebpf/README.md @@ -0,0 +1,32 @@ +# syscall_ebpf + +## Prerequisites + +1. Install bpf-linker: `cargo install bpf-linker` + +## Build eBPF + +```bash +cargo xtask build-ebpf +``` + +To perform a release build you can use the `--release` flag. +You may also change the target architecture with the `--target` flag. + +## Build Userspace + +```bash +cargo build +``` + +## Build eBPF and Userspace + +```bash +cargo xtask build +``` + +## Run + +```bash +RUST_LOG=info cargo xtask run +``` diff --git a/user/apps/test_ebpf/syscall_ebpf/syscall_ebpf-common/Cargo.toml b/user/apps/test_ebpf/syscall_ebpf/syscall_ebpf-common/Cargo.toml new file mode 100644 index 000000000..7acc25d40 --- /dev/null +++ b/user/apps/test_ebpf/syscall_ebpf/syscall_ebpf-common/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "syscall_ebpf-common" +version = "0.1.0" +edition = "2021" + +[features] +default = [] +user = ["aya"] + +[dependencies] +aya = { git = "https://git.mirrors.dragonos.org.cn/DragonOS-Community/tiny-aya.git", rev = "0689f13", optional = true } + +[lib] +path = "src/lib.rs" diff --git a/user/apps/test_ebpf/syscall_ebpf/syscall_ebpf-common/src/lib.rs b/user/apps/test_ebpf/syscall_ebpf/syscall_ebpf-common/src/lib.rs new file mode 100644 index 000000000..0c9ac1ac8 --- /dev/null +++ b/user/apps/test_ebpf/syscall_ebpf/syscall_ebpf-common/src/lib.rs @@ -0,0 +1 @@ +#![no_std] diff --git a/user/apps/test_ebpf/syscall_ebpf/syscall_ebpf-ebpf/.cargo/config.toml b/user/apps/test_ebpf/syscall_ebpf/syscall_ebpf-ebpf/.cargo/config.toml new file mode 100644 index 000000000..4302a7f16 --- /dev/null +++ b/user/apps/test_ebpf/syscall_ebpf/syscall_ebpf-ebpf/.cargo/config.toml @@ -0,0 +1,6 @@ +[build] +target-dir = "../target" +target = "bpfel-unknown-none" + +[unstable] +build-std = ["core"] diff --git a/user/apps/test_ebpf/syscall_ebpf/syscall_ebpf-ebpf/.helix/config.toml b/user/apps/test_ebpf/syscall_ebpf/syscall_ebpf-ebpf/.helix/config.toml new file mode 100644 index 000000000..da5424f19 --- /dev/null +++ b/user/apps/test_ebpf/syscall_ebpf/syscall_ebpf-ebpf/.helix/config.toml @@ -0,0 +1,2 @@ +[editor] +workspace-lsp-roots = [] diff --git a/user/apps/test_ebpf/syscall_ebpf/syscall_ebpf-ebpf/.vim/coc-settings.json b/user/apps/test_ebpf/syscall_ebpf/syscall_ebpf-ebpf/.vim/coc-settings.json new file mode 100644 index 000000000..e2211a64f --- /dev/null +++ b/user/apps/test_ebpf/syscall_ebpf/syscall_ebpf-ebpf/.vim/coc-settings.json @@ -0,0 +1,4 @@ +{ + "rust-analyzer.cargo.target": "bpfel-unknown-none", + "rust-analyzer.checkOnSave.allTargets": false +} diff --git a/user/apps/test_ebpf/syscall_ebpf/syscall_ebpf-ebpf/.vscode/settings.json b/user/apps/test_ebpf/syscall_ebpf/syscall_ebpf-ebpf/.vscode/settings.json new file mode 100644 index 000000000..e2211a64f --- /dev/null +++ b/user/apps/test_ebpf/syscall_ebpf/syscall_ebpf-ebpf/.vscode/settings.json @@ -0,0 +1,4 @@ +{ + "rust-analyzer.cargo.target": "bpfel-unknown-none", + "rust-analyzer.checkOnSave.allTargets": false +} diff --git a/user/apps/test_ebpf/syscall_ebpf/syscall_ebpf-ebpf/Cargo.toml b/user/apps/test_ebpf/syscall_ebpf/syscall_ebpf-ebpf/Cargo.toml new file mode 100644 index 000000000..1911fa43c --- /dev/null +++ b/user/apps/test_ebpf/syscall_ebpf/syscall_ebpf-ebpf/Cargo.toml @@ -0,0 +1,33 @@ +[package] +name = "syscall_ebpf-ebpf" +version = "0.1.0" +edition = "2021" + +[dependencies] +aya-ebpf = { git = "https://git.mirrors.dragonos.org.cn/DragonOS-Community/aya.git", rev = "3d57d35" } +aya-log-ebpf = { git = "https://git.mirrors.dragonos.org.cn/DragonOS-Community/aya.git", rev = "3d57d35" } + +syscall_ebpf-common = { path = "../syscall_ebpf-common" } + +[[bin]] +name = "syscall_ebpf" +path = "src/main.rs" + +[profile.dev] +opt-level = 3 +debug = false +debug-assertions = false +overflow-checks = false +lto = true +panic = "abort" +incremental = false +codegen-units = 1 +rpath = false + +[profile.release] +lto = true +panic = "abort" +codegen-units = 1 + +[workspace] +members = [] diff --git a/user/apps/test_ebpf/syscall_ebpf/syscall_ebpf-ebpf/rust-toolchain.toml b/user/apps/test_ebpf/syscall_ebpf/syscall_ebpf-ebpf/rust-toolchain.toml new file mode 100644 index 000000000..447b8f049 --- /dev/null +++ b/user/apps/test_ebpf/syscall_ebpf/syscall_ebpf-ebpf/rust-toolchain.toml @@ -0,0 +1,13 @@ +[toolchain] +channel = "nightly-2024-07-23" +# The source code of rustc, provided by the rust-src component, is needed for +# building eBPF programs. +components = [ + "cargo", + "clippy", + "rust-docs", + "rust-src", + "rust-std", + "rustc", + "rustfmt", +] diff --git a/user/apps/test_ebpf/syscall_ebpf/syscall_ebpf-ebpf/src/main.rs b/user/apps/test_ebpf/syscall_ebpf/syscall_ebpf-ebpf/src/main.rs new file mode 100644 index 000000000..7f9b79b65 --- /dev/null +++ b/user/apps/test_ebpf/syscall_ebpf/syscall_ebpf-ebpf/src/main.rs @@ -0,0 +1,44 @@ +#![no_std] +#![no_main] + +use aya_ebpf::{macros::kprobe, programs::ProbeContext}; +use aya_ebpf::macros::map; +use aya_ebpf::maps::HashMap; +use aya_log_ebpf::info; + +#[kprobe] +pub fn syscall_ebpf(ctx: ProbeContext) -> u32 { + try_syscall_ebpf(ctx).unwrap_or_else(|ret| ret) +} + +fn try_syscall_ebpf(ctx: ProbeContext) -> Result { + let pt_regs = unsafe { + &*ctx.regs + }; + // first arg -> rdi + // second arg -> rsi + // third arg -> rdx + // four arg -> rcx + let syscall_num = pt_regs.rsi as usize; + if syscall_num != 1 { + unsafe { + if let Some(v) = SYSCALL_LIST.get(&(syscall_num as u32)){ + let new_v = *v + 1; + SYSCALL_LIST.insert(&(syscall_num as u32), &new_v,0).unwrap(); + }else { + SYSCALL_LIST.insert(&(syscall_num as u32), &1,0).unwrap(); + } + } + info!(&ctx, "invoke syscall {}", syscall_num); + } + Ok(0) +} + +#[map] // +static SYSCALL_LIST: HashMap = + HashMap::::with_max_entries(1024, 0); + +#[panic_handler] +fn panic(_info: &core::panic::PanicInfo) -> ! { + unsafe { core::hint::unreachable_unchecked() } +} diff --git a/user/apps/test_ebpf/syscall_ebpf/xtask/Cargo.toml b/user/apps/test_ebpf/syscall_ebpf/xtask/Cargo.toml new file mode 100644 index 000000000..c4dea5d16 --- /dev/null +++ b/user/apps/test_ebpf/syscall_ebpf/xtask/Cargo.toml @@ -0,0 +1,8 @@ +[package] +name = "xtask" +version = "0.1.0" +edition = "2021" + +[dependencies] +anyhow = "1" +clap = { version = "4.1", features = ["derive"] } diff --git a/user/apps/test_ebpf/syscall_ebpf/xtask/src/build.rs b/user/apps/test_ebpf/syscall_ebpf/xtask/src/build.rs new file mode 100644 index 000000000..ddeee4496 --- /dev/null +++ b/user/apps/test_ebpf/syscall_ebpf/xtask/src/build.rs @@ -0,0 +1,42 @@ +use std::process::Command; + +use anyhow::Context as _; +use clap::Parser; + +use crate::build_ebpf::{build_ebpf, Architecture, Options as BuildOptions}; + +#[derive(Debug, Parser)] +pub struct Options { + /// Set the endianness of the BPF target + #[clap(default_value = "bpfel-unknown-none", long)] + pub bpf_target: Architecture, + /// Build and run the release target + #[clap(long)] + pub release: bool, +} + +/// Build the project +fn build_project(opts: &Options) -> Result<(), anyhow::Error> { + let mut args = vec!["build"]; + if opts.release { + args.push("--release") + } + let status = Command::new("cargo") + .args(&args) + .status() + .expect("failed to build userspace"); + assert!(status.success()); + Ok(()) +} + +/// Build our ebpf program and the project +pub fn build(opts: Options) -> Result<(), anyhow::Error> { + // build our ebpf program followed by our application + build_ebpf(BuildOptions { + target: opts.bpf_target, + release: opts.release, + }) + .context("Error while building eBPF program")?; + build_project(&opts).context("Error while building userspace application")?; + Ok(()) +} \ No newline at end of file diff --git a/user/apps/test_ebpf/syscall_ebpf/xtask/src/build_ebpf.rs b/user/apps/test_ebpf/syscall_ebpf/xtask/src/build_ebpf.rs new file mode 100644 index 000000000..8c6e323f5 --- /dev/null +++ b/user/apps/test_ebpf/syscall_ebpf/xtask/src/build_ebpf.rs @@ -0,0 +1,67 @@ +use std::{path::PathBuf, process::Command}; + +use clap::Parser; + +#[derive(Debug, Copy, Clone)] +pub enum Architecture { + BpfEl, + BpfEb, +} + +impl std::str::FromStr for Architecture { + type Err = String; + + fn from_str(s: &str) -> Result { + Ok(match s { + "bpfel-unknown-none" => Architecture::BpfEl, + "bpfeb-unknown-none" => Architecture::BpfEb, + _ => return Err("invalid target".to_owned()), + }) + } +} + +impl std::fmt::Display for Architecture { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(match self { + Architecture::BpfEl => "bpfel-unknown-none", + Architecture::BpfEb => "bpfeb-unknown-none", + }) + } +} + +#[derive(Debug, Parser)] +pub struct Options { + /// Set the endianness of the BPF target + #[clap(default_value = "bpfel-unknown-none", long)] + pub target: Architecture, + /// Build the release target + #[clap(long)] + pub release: bool, +} + +pub fn build_ebpf(opts: Options) -> Result<(), anyhow::Error> { + let dir = PathBuf::from("syscall_ebpf-ebpf"); + let target = format!("--target={}", opts.target); + let mut args = vec![ + "build", + target.as_str(), + "-Z", + "build-std=core", + ]; + if opts.release { + args.push("--release") + } + + // Command::new creates a child process which inherits all env variables. This means env + // vars set by the cargo xtask command are also inherited. RUSTUP_TOOLCHAIN is removed + // so the rust-toolchain.toml file in the -ebpf folder is honored. + + let status = Command::new("cargo") + .current_dir(dir) + .env_remove("RUSTUP_TOOLCHAIN") + .args(&args) + .status() + .expect("failed to build bpf program"); + assert!(status.success()); + Ok(()) +} diff --git a/user/apps/test_ebpf/syscall_ebpf/xtask/src/main.rs b/user/apps/test_ebpf/syscall_ebpf/xtask/src/main.rs new file mode 100644 index 000000000..507945899 --- /dev/null +++ b/user/apps/test_ebpf/syscall_ebpf/xtask/src/main.rs @@ -0,0 +1,36 @@ +mod build_ebpf; +mod build; +mod run; + +use std::process::exit; + +use clap::Parser; + +#[derive(Debug, Parser)] +pub struct Options { + #[clap(subcommand)] + command: Command, +} + +#[derive(Debug, Parser)] +enum Command { + BuildEbpf(build_ebpf::Options), + Build(build::Options), + Run(run::Options), +} + +fn main() { + let opts = Options::parse(); + + use Command::*; + let ret = match opts.command { + BuildEbpf(opts) => build_ebpf::build_ebpf(opts), + Run(opts) => run::run(opts), + Build(opts) => build::build(opts), + }; + + if let Err(e) = ret { + eprintln!("{e:#}"); + exit(1); + } +} diff --git a/user/apps/test_ebpf/syscall_ebpf/xtask/src/run.rs b/user/apps/test_ebpf/syscall_ebpf/xtask/src/run.rs new file mode 100644 index 000000000..19af11c45 --- /dev/null +++ b/user/apps/test_ebpf/syscall_ebpf/xtask/src/run.rs @@ -0,0 +1,55 @@ +use std::process::Command; + +use anyhow::Context as _; +use clap::Parser; + +use crate::{build::{build, Options as BuildOptions}, build_ebpf::Architecture}; + +#[derive(Debug, Parser)] +pub struct Options { + /// Set the endianness of the BPF target + #[clap(default_value = "bpfel-unknown-none", long)] + pub bpf_target: Architecture, + /// Build and run the release target + #[clap(long)] + pub release: bool, + /// The command used to wrap your application + #[clap(short, long, default_value = "sudo -E")] + pub runner: String, + /// Arguments to pass to your application + #[clap(name = "args", last = true)] + pub run_args: Vec, +} + + +/// Build and run the project +pub fn run(opts: Options) -> Result<(), anyhow::Error> { + // Build our ebpf program and the project + build(BuildOptions{ + bpf_target: opts.bpf_target, + release: opts.release, + }).context("Error while building project")?; + + // profile we are building (release or debug) + let profile = if opts.release { "release" } else { "debug" }; + let bin_path = format!("target/{profile}/syscall_ebpf"); + + // arguments to pass to the application + let mut run_args: Vec<_> = opts.run_args.iter().map(String::as_str).collect(); + + // configure args + let mut args: Vec<_> = opts.runner.trim().split_terminator(' ').collect(); + args.push(bin_path.as_str()); + args.append(&mut run_args); + + // run the command + let status = Command::new(args.first().expect("No first argument")) + .args(args.iter().skip(1)) + .status() + .expect("failed to run the command"); + + if !status.success() { + anyhow::bail!("Failed to run `{}`", args.join(" ")); + } + Ok(()) +} diff --git a/user/apps/test_tokio/src/main.rs b/user/apps/test_tokio/src/main.rs index 91b06613b..cded478d4 100644 --- a/user/apps/test_tokio/src/main.rs +++ b/user/apps/test_tokio/src/main.rs @@ -1,5 +1,3 @@ -use tokio::signal; - async fn say_world() { println!("world"); } diff --git a/user/dadk/config/test_ebpf_0_1_0.dadk b/user/dadk/config/test_ebpf_0_1_0.dadk new file mode 100644 index 000000000..250952d32 --- /dev/null +++ b/user/dadk/config/test_ebpf_0_1_0.dadk @@ -0,0 +1,23 @@ +{ + "name": "test_ebpf", + "version": "0.1.0", + "description": "to test eBPF", + "task_type": { + "BuildFromSource": { + "Local": { + "path": "apps/test_ebpf" + } + } + }, + "depends": [], + "build": { + "build_command": "make install" + }, + "install": { + "in_dragonos_path": "/" + }, + "clean": { + "clean_command": "make clean" + }, + "target_arch": ["x86_64"] +} From 37c2359ac400be9a77c26542bd60ef8267aa5ee6 Mon Sep 17 00:00:00 2001 From: LoGin Date: Fri, 25 Oct 2024 18:59:54 +0800 Subject: [PATCH 09/30] =?UTF-8?q?ci:=20=E5=88=A0=E9=99=A4PR=20#948=20?= =?UTF-8?q?=E4=BB=A3=E5=85=A5=E7=9A=84cache-toolchain.toml=20(#1012)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/cache-toolchain.yml | 94 --------------------------- 1 file changed, 94 deletions(-) delete mode 100644 .github/workflows/cache-toolchain.yml diff --git a/.github/workflows/cache-toolchain.yml b/.github/workflows/cache-toolchain.yml deleted file mode 100644 index 4b6099c54..000000000 --- a/.github/workflows/cache-toolchain.yml +++ /dev/null @@ -1,94 +0,0 @@ -name: Cache toolchain - -on: workflow_call - -jobs: - build: - - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v3 - - - name: Cache DragonOS GCC - id: cache-dragonos-gcc - uses: actions/cache@v3 - env: - cache-name: cache-dragonos-gcc - with: - path: | - ~/opt - ~/.bashrc - key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('tools/build_gcc_toolchain.sh') }} - - - if: ${{ steps.cache-dragonos-gcc.outputs.cache-hit != 'true' }} - name: build dragonos-gcc - continue-on-error: true - run: | - bash tools/build_gcc_toolchain.sh -f - - - uses: ./.github/actions/install-apt-packages - - - name: Cache build tools - id: cache-build-tools - uses: actions/cache@v3 - env: - cache-name: cache-build-tools - dadk_version: 0.1.11 - with: - path: | - ~/.cargo - ~/.rustup - ~/.bashrc - ~/opt - key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ env.dadk_version }}-${{ hashFiles('.github/workflows/cache-toolchain.yml') }}-${{ hashFiles('tools/install_musl_gcc.sh') }} - - - if: ${{ steps.cache-build-tools.outputs.cache-hit != 'true' }} - name: Install toolchain - continue-on-error: false - run: | - USE_GITHUB=1 bash tools/install_musl_gcc.sh - - cargo install cargo-binutils - rustup toolchain install nightly-x86_64-unknown-linux-gnu - rustup toolchain install nightly-2024-07-23-x86_64-unknown-linux-gnu - rustup toolchain install nightly-2023-08-15-x86_64-unknown-linux-gnu - rustup component add rust-src --toolchain nightly-2024-07-23-x86_64-unknown-linux-gnu - rustup component add rust-src --toolchain nightly-2023-08-15-x86_64-unknown-linux-gnu - rustup target add x86_64-unknown-none --toolchain nightly-2024-07-23-x86_64-unknown-linux-gnu - rustup target add x86_64-unknown-none --toolchain nightly-2023-08-15-x86_64-unknown-linux-gnu - - rustup toolchain install nightly-2024-07-23-riscv64gc-unknown-linux-gnu --force-non-host - rustup toolchain install nightly-2023-08-15-riscv64gc-unknown-linux-gnu --force-non-host - rustup target add riscv64gc-unknown-none-elf --toolchain nightly-2024-07-23-riscv64gc-unknown-linux-gnu - rustup target add riscv64imac-unknown-none-elf --toolchain nightly-2024-07-23-riscv64gc-unknown-linux-gnu - rustup target add riscv64gc-unknown-none-elf --toolchain nightly-2023-08-15-riscv64gc-unknown-linux-gnu - rustup target add riscv64imac-unknown-none-elf --toolchain nightly-2023-08-15-riscv64gc-unknown-linux-gnu - - rustup component add rust-src --toolchain nightly-x86_64-unknown-linux-gnu - rustup component add rust-src - rustup component add llvm-tools-preview - - rustup component add rustfmt - rustup component add rustfmt --toolchain nightly-x86_64-unknown-linux-gnu - rustup component add rustfmt --toolchain nightly-2024-07-23-x86_64-unknown-linux-gnu - rustup component add rustfmt --toolchain nightly-2023-08-15-x86_64-unknown-linux-gnu - rustup component add rustfmt --toolchain nightly-2024-07-23-riscv64gc-unknown-linux-gnu - rustup component add rustfmt --toolchain nightly-2023-08-15-riscv64gc-unknown-linux-gnu - - rustup default nightly-2024-07-23 - - cargo install dadk --version 0.1.11 - - - userapp_musl_toolchain="nightly-2023-08-15-x86_64-unknown-linux-gnu" - - rustup toolchain install ${userapp_musl_toolchain} - rustup component add --toolchain ${userapp_musl_toolchain} rust-src - rustup target add --toolchain ${userapp_musl_toolchain} x86_64-unknown-linux-musl - - rustup target add x86_64-unknown-linux-musl --toolchain nightly-2024-07-23-x86_64-unknown-linux-gnu - rustup component add rust-src --toolchain nightly-2024-07-23-x86_64-unknown-linux-gnu - cargo install bpf-linker - - From dc60d68f0ed40581959a5d98d01ef8a87b776f84 Mon Sep 17 00:00:00 2001 From: LoGin Date: Sat, 26 Oct 2024 12:55:06 +0800 Subject: [PATCH 10/30] =?UTF-8?q?fix:=20tty=E5=88=B7=E6=96=B0=E7=BA=BF?= =?UTF-8?q?=E7=A8=8B=E5=88=9D=E5=A7=8B=E5=8C=96=E4=B9=8B=E5=89=8D=EF=BC=8C?= =?UTF-8?q?=E9=94=AE=E7=9B=98=E4=BA=A7=E7=94=9F=E6=95=B0=E6=8D=AE=E5=AF=BC?= =?UTF-8?q?=E8=87=B4=E5=B4=A9=E6=BA=83=E7=9A=84=E9=97=AE=E9=A2=98=20(#1013?= =?UTF-8?q?)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: longjin --- kernel/src/driver/tty/kthread.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/kernel/src/driver/tty/kthread.rs b/kernel/src/driver/tty/kthread.rs index 9016b0d1d..28b2fa2b0 100644 --- a/kernel/src/driver/tty/kthread.rs +++ b/kernel/src/driver/tty/kthread.rs @@ -62,6 +62,10 @@ fn tty_refresh_thread() -> i32 { /// 发送数据到tty刷新线程 pub fn send_to_tty_refresh_thread(data: &[u8]) { + if unsafe { TTY_REFRESH_THREAD.is_none() } { + return; + } + for item in data { KEYBUF.push(*item).ok(); } From a9e28e9ce9607ebd1953898dda370c22d4f1425d Mon Sep 17 00:00:00 2001 From: LoGin Date: Sat, 26 Oct 2024 12:55:19 +0800 Subject: [PATCH 11/30] =?UTF-8?q?fix:=20=E4=B8=B4=E6=97=B6=E4=BF=AE?= =?UTF-8?q?=E5=A4=8D=E9=94=AE=E7=9B=98=E9=A9=B1=E5=8A=A8=E4=B8=8E=E9=BC=A0?= =?UTF-8?q?=E6=A0=87=E9=A9=B1=E5=8A=A8=E5=86=B2=E7=AA=81=E5=AF=BC=E8=87=B4?= =?UTF-8?q?=E9=94=AE=E7=9B=98=E6=97=A0=E5=93=8D=E5=BA=94=20(#1014)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 暂时通过条件编译的方式解决. 目前认为是鼠标驱动问题,没有正确判断是不是自己的数据... 但是因为我们场景下,鼠标驱动几乎用不到,因此先条件编译屏蔽. Signed-off-by: longjin --- kernel/Cargo.toml | 1 + kernel/src/arch/x86_64/asm/entry.S | 1 - kernel/src/driver/input/mod.rs | 2 +- kernel/src/driver/input/serio/i8042/mod.rs | 2 +- kernel/src/driver/keyboard/ps2_keyboard.rs | 59 +++++++++++++++------- 5 files changed, 44 insertions(+), 21 deletions(-) diff --git a/kernel/Cargo.toml b/kernel/Cargo.toml index ebe69b99b..d18d22f02 100644 --- a/kernel/Cargo.toml +++ b/kernel/Cargo.toml @@ -23,6 +23,7 @@ kvm = [] fatfs = [] fatfs-secure = ["fatfs"] +driver_ps2_mouse = [] # kprobe kprobe_test = [] diff --git a/kernel/src/arch/x86_64/asm/entry.S b/kernel/src/arch/x86_64/asm/entry.S index f61d74725..355d9938f 100644 --- a/kernel/src/arch/x86_64/asm/entry.S +++ b/kernel/src/arch/x86_64/asm/entry.S @@ -54,7 +54,6 @@ Restore_all: popq %rax addq $0x10, %rsp // 弹出变量FUNC和errcode - sti iretq ret_from_exception: diff --git a/kernel/src/driver/input/mod.rs b/kernel/src/driver/input/mod.rs index 3bd4c7eec..1067a1e8a 100644 --- a/kernel/src/driver/input/mod.rs +++ b/kernel/src/driver/input/mod.rs @@ -1,4 +1,4 @@ pub mod ps2_dev; -#[cfg(target_arch = "x86_64")] +#[cfg(all(target_arch = "x86_64", feature = "driver_ps2_mouse"))] pub mod ps2_mouse; pub mod serio; diff --git a/kernel/src/driver/input/serio/i8042/mod.rs b/kernel/src/driver/input/serio/i8042/mod.rs index b70873ae9..a54d42816 100644 --- a/kernel/src/driver/input/serio/i8042/mod.rs +++ b/kernel/src/driver/input/serio/i8042/mod.rs @@ -69,7 +69,7 @@ pub fn i8042_setup_aux() -> Result<(), SystemError> { ))); serio_device_manager().register_port(aux_port.clone() as Arc)?; - #[cfg(target_arch = "x86_64")] + #[cfg(all(target_arch = "x86_64", feature = "driver_ps2_mouse"))] crate::driver::input::ps2_mouse::ps_mouse_device::rs_ps2_mouse_device_init( aux_port.clone() as Arc )?; diff --git a/kernel/src/driver/keyboard/ps2_keyboard.rs b/kernel/src/driver/keyboard/ps2_keyboard.rs index 4133be0b2..94564e9fb 100644 --- a/kernel/src/driver/keyboard/ps2_keyboard.rs +++ b/kernel/src/driver/keyboard/ps2_keyboard.rs @@ -8,7 +8,7 @@ use alloc::{ use unified_init::macros::unified_init; use crate::{ - arch::{io::PortIOArch, CurrentPortIOArch}, + arch::{io::PortIOArch, CurrentIrqArch, CurrentPortIOArch}, driver::{ base::device::device_number::{DeviceNumber, Major}, input::ps2_dev::Ps2StatusRegister, @@ -17,7 +17,7 @@ use crate::{ irqdata::IrqHandlerData, irqdesc::{IrqHandleFlags, IrqHandler, IrqReturn}, manage::irq_manager, - IrqNumber, + InterruptArch, IrqNumber, }, filesystem::{ devfs::{devfs_register, DevFS, DeviceINode}, @@ -186,17 +186,24 @@ impl IrqHandler for Ps2KeyboardIrqHandler { _dev_id: Option>, ) -> Result { // 先检查状态寄存器，看看是否有数据 - let status = unsafe { CurrentPortIOArch::in8(PORT_PS2_KEYBOARD_STATUS.into()) }; - let status = Ps2StatusRegister::from(status); - if !status.outbuf_full() { - return Ok(IrqReturn::Handled); + let mut fsm = PS2_KEYBOARD_FSM.lock(); + let mut handled = false; + loop { + let status = unsafe { CurrentPortIOArch::in8(PORT_PS2_KEYBOARD_STATUS.into()) }; + let status = Ps2StatusRegister::from(status); + if status.outbuf_full() { + let input = unsafe { CurrentPortIOArch::in8(PORT_PS2_KEYBOARD_DATA.into()) }; + fsm.parse(input); + handled = true; + } else { + break; + } + } + if handled { + Ok(IrqReturn::Handled) + } else { + Ok(IrqReturn::NotHandled) } - - let input = unsafe { CurrentPortIOArch::in8(PORT_PS2_KEYBOARD_DATA.into()) }; - // wait_ps2_keyboard_read(); - PS2_KEYBOARD_FSM.lock().parse(input); - - return Ok(IrqReturn::Handled); } } @@ -219,8 +226,27 @@ fn wait_ps2_keyboard_write() { spin_loop(); } } + +fn force_clear_input_buffer() { + loop { + let status = unsafe { CurrentPortIOArch::in8(PORT_PS2_KEYBOARD_STATUS.into()) }; + let status = Ps2StatusRegister::from(status); + if status.outbuf_full() { + unsafe { CurrentPortIOArch::in8(PORT_PS2_KEYBOARD_DATA.into()) }; + } else { + break; + } + } +} + #[unified_init(INITCALL_DEVICE)] fn ps2_keyboard_init() -> Result<(), SystemError> { + let irq_guard = unsafe { CurrentIrqArch::save_and_disable_irq() }; + + // 先读一下键盘的数据，防止由于在键盘初始化之前，由于按键被按下从而导致接收不到中断。 + // 清空键盘缓冲区 + force_clear_input_buffer(); + // ======== 初始化键盘控制器，写入配置值 ========= wait_ps2_keyboard_write(); unsafe { @@ -245,12 +271,9 @@ fn ps2_keyboard_init() -> Result<(), SystemError> { ) .expect("Failed to request irq for ps2 keyboard"); - // 先读一下键盘的数据，防止由于在键盘初始化之前，由于按键被按下从而导致接收不到中断。 - let status = unsafe { CurrentPortIOArch::in8(PORT_PS2_KEYBOARD_STATUS.into()) }; - let status = Ps2StatusRegister::from(status); - if status.outbuf_full() { - unsafe { CurrentPortIOArch::in8(PORT_PS2_KEYBOARD_DATA.into()) }; - } + // 清空键盘缓冲区 + force_clear_input_buffer(); + drop(irq_guard); // 将设备挂载到devfs ps2_keyboard_register(); From 4dd4856f933be0b4624c7f7ffa9e3d0c8c218873 Mon Sep 17 00:00:00 2001 From: LoGin Date: Sat, 26 Oct 2024 12:55:31 +0800 Subject: [PATCH 12/30] =?UTF-8?q?fix(tty):=20=E4=BF=AE=E5=A4=8Dtty?= =?UTF-8?q?=E5=AF=B9tab=E8=BF=9B=E8=A1=8C=E5=A4=84=E7=90=86=E6=97=B6?= =?UTF-8?q?=E4=BA=A7=E7=94=9F=E6=95=B0=E7=BB=84=E8=B6=8A=E7=95=8Cpanic?= =?UTF-8?q?=E7=9A=84=E9=97=AE=E9=A2=98=20(#1015)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: longjin --- .../tty/virtual_terminal/virtual_console.rs | 21 ++++++++----------- kernel/src/libs/font/mod.rs | 2 +- 2 files changed, 10 insertions(+), 13 deletions(-) diff --git a/kernel/src/driver/tty/virtual_terminal/virtual_console.rs b/kernel/src/driver/tty/virtual_terminal/virtual_console.rs index da7cdcd50..773d99c4b 100644 --- a/kernel/src/driver/tty/virtual_terminal/virtual_console.rs +++ b/kernel/src/driver/tty/virtual_terminal/virtual_console.rs @@ -1284,13 +1284,11 @@ impl VirtualConsoleData { // 水平制表符（Horizontal Tab） self.pos -= self.state.x; - let ret = self.tab_stop.next_index(self.state.x + 1); - - if let Some(x) = ret { - self.state.x = x; - } else { - self.state.x = self.cols - 1; - } + self.state.x = self + .tab_stop + .next_index(self.state.x + 1) + .unwrap_or(self.cols - 1); + self.state.x = core::cmp::min(self.state.x, self.cols - 1); self.pos += self.state.x; // TODO: notify @@ -1552,7 +1550,7 @@ impl VirtualConsoleData { } // 未找到 - if (!self.utf || self.display_ctrl || c < 128) && c & !charmask == 0 { + if (!self.utf || self.display_ctrl || c < 128) && (c & !charmask) == 0 { tc = c; } else { let tmp = self.unicode_to_index(0xfffd); @@ -1587,7 +1585,7 @@ impl VirtualConsoleData { // TODO: 处理unicode screen buf if himask != 0 { - tc = (if tc & 0x100 != 0 { himask as u32 } else { 0 }) | (tc & 0xff); + tc = (if (tc & 0x100) != 0 { himask as u32 } else { 0 }) | (tc & 0xff); } tc |= ((attr as u32) << 8) & (!himask as u32); @@ -1602,7 +1600,7 @@ impl VirtualConsoleData { // ); self.screen_buf[self.pos] = tc as u16; - if draw.x.is_none() { + if self.should_update() && draw.x.is_none() { // 设置draw参数 draw.x = Some(self.state.x as u32); draw.offset = self.pos; @@ -1619,7 +1617,7 @@ impl VirtualConsoleData { } width -= 1; - if width == 0 { + if width <= 0 { break; } let tmp = self.unicode_to_index(' ' as u32); @@ -1629,7 +1627,6 @@ impl VirtualConsoleData { if invert { self.flush(draw); } - true } diff --git a/kernel/src/libs/font/mod.rs b/kernel/src/libs/font/mod.rs index 90eeeb354..d059b0d98 100644 --- a/kernel/src/libs/font/mod.rs +++ b/kernel/src/libs/font/mod.rs @@ -40,7 +40,7 @@ impl FontDesc { } for (first, last) in Self::DOUBLE_WIDTH_RANGE { - if ch > *first && ch < *last { + if ch >= *first && ch < *last { return true; } } From 7f46e98e36ef30309306b48067e77795fb92308c Mon Sep 17 00:00:00 2001 From: LoGin Date: Sat, 26 Oct 2024 17:14:41 +0800 Subject: [PATCH 13/30] =?UTF-8?q?chore:=20=E4=BF=AE=E6=94=B9traigebot?= =?UTF-8?q?=E9=85=8D=E7=BD=AE=EF=BC=88=E5=85=B3=E4=BA=8Esig-obs=20&=20Test?= =?UTF-8?q?=EF=BC=89=20(#1016)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: longjin --- triagebot.toml | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/triagebot.toml b/triagebot.toml index d723edb01..ffc4dfc3d 100644 --- a/triagebot.toml +++ b/triagebot.toml @@ -82,6 +82,14 @@ trigger_files = [ [autolabel."T-virtulization"] trigger_files = ["kernel/src/virt", "kernel/src/arch/x86_64/kvm"] +[autolabel."T-Obs and Test"] +trigger_files = [ + "/kernel/crates/rbpf", + "/kernel/crates/kprobe", + "/kernel/src/debug/kprobe", + "/kernel/src/bpf", +] + [autolabel."S-等待审查"] new_pr = true @@ -107,6 +115,10 @@ cc = ["@fslongjin"] message = "tty驱动发生了更改，请进行审查" cc = ["@GnoCiYeH"] +[mentions."kernel/src/bpf"] +message = "BPF部分发生了更改，请进行审查" +cc = ["@Godones"] + [assign] warn_non_default_branch = true @@ -121,6 +133,13 @@ driver = ["@dragonos/main"] # SIG-MM sig-mm = ["@dragonos/mm"] +sig-obs = [ + "@dragonos/sig-observation-testing", + "@Chiichen", + "@Godones" +] + + # 虚拟化 virtulization = ["@dragonos/virtualization"] @@ -148,4 +167,8 @@ infra = ["@dragonos/infra"] "/kernel/src/arch/x86_64/kvm" = ["virtulization"] "/kernel/src/arch/x86_64" = ["x86_64"] "/kernel/src/arch/riscv64" = ["riscv64"] +"/kernel/crates/rbpf" = ["sig-obs"] +"/kernel/crates/kprobe" = ["sig-obs"] +"/kernel/src/debug/kprobe" = ["sig-obs"] +"/kernel/src/bpf" = ["sig-obs"] "/tools" = ["infra"] From 55e6f0b65f91b32638fd56581f711a816eccdcd1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=81=AB=E8=8A=B1?= Date: Sat, 26 Oct 2024 18:13:25 +0800 Subject: [PATCH 14/30] =?UTF-8?q?feat:=E6=B7=BB=E5=8A=A0chown=E7=B3=BB?= =?UTF-8?q?=E7=BB=9F=E8=B0=83=E7=94=A8=20(#962)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 添加chown系统调用 --------- Co-authored-by: sparkzky Co-authored-by: longjin --- kernel/src/filesystem/vfs/mod.rs | 62 +++++++--- kernel/src/filesystem/vfs/open.rs | 89 +++++++++++++- kernel/src/filesystem/vfs/syscall.rs | 50 +++++++- kernel/src/process/cred.rs | 2 +- kernel/src/syscall/mod.rs | 28 ++++- user/apps/test-chown/.gitignore | 4 + user/apps/test-chown/Cargo.toml | 13 ++ user/apps/test-chown/Makefile | 56 +++++++++ user/apps/test-chown/README.md | 8 ++ user/apps/test-chown/src/main.rs | 160 +++++++++++++++++++++++++ user/dadk/config/test_chown_0_1_0.dadk | 29 +++++ 11 files changed, 481 insertions(+), 20 deletions(-) create mode 100644 user/apps/test-chown/.gitignore create mode 100644 user/apps/test-chown/Cargo.toml create mode 100644 user/apps/test-chown/Makefile create mode 100644 user/apps/test-chown/README.md create mode 100644 user/apps/test-chown/src/main.rs create mode 100644 user/dadk/config/test_chown_0_1_0.dadk diff --git a/kernel/src/filesystem/vfs/mod.rs b/kernel/src/filesystem/vfs/mod.rs index 51d2c4220..7801258e2 100644 --- a/kernel/src/filesystem/vfs/mod.rs +++ b/kernel/src/filesystem/vfs/mod.rs @@ -598,17 +598,43 @@ impl dyn IndexNode { return self.lookup_follow_symlink(path, 0); } - /// @brief 查找文件（考虑符号链接） + pub fn lookup_follow_symlink( + &self, + path: &str, + max_follow_times: usize, + ) -> Result, SystemError> { + return self.do_lookup_follow_symlink(path, max_follow_times, true); + } + + pub fn lookup_follow_symlink2( + &self, + path: &str, + max_follow_times: usize, + follow_final_symlink: bool, + ) -> Result, SystemError> { + return self.do_lookup_follow_symlink(path, max_follow_times, follow_final_symlink); + } + + /// # 查找文件 + /// 查找指定路径的文件，考虑符号链接的存在，并可选择是否返回最终路径的符号链接文件本身。 /// - /// @param path 文件路径 - /// @param max_follow_times 最大经过的符号链接的大小 + /// ## 参数 + /// - `path`: 文件路径 + /// - `max_follow_times`: 最大经过的符号链接的数量 + /// - `follow_final_symlink`: 是否跟随最后的符号链接 /// - /// @return Ok(Arc) 要寻找的目录项的inode - /// @return Err(SystemError) 错误码 - pub fn lookup_follow_symlink( + /// ## 返回值 + /// - `Ok(Arc)`: 要寻找的目录项的inode + /// - `Err(SystemError)`: 错误码，表示查找过程中遇到的错误 + /// + /// ## Safety + /// 此函数在处理符号链接时可能会遇到循环引用的情况，`max_follow_times` 参数用于限制符号链接的跟随次数以避免无限循环。 + #[inline(never)] + pub fn do_lookup_follow_symlink( &self, path: &str, max_follow_times: usize, + follow_final_symlink: bool, ) -> Result, SystemError> { if self.metadata()?.file_type != FileType::Dir { return Err(SystemError::ENOTDIR); @@ -632,13 +658,10 @@ impl dyn IndexNode { } let name; - // 寻找“/” match rest_path.find('/') { Some(pos) => { - // 找到了，设置下一个要查找的名字 name = String::from(&rest_path[0..pos]); - // 剩余的路径字符串 rest_path = String::from(&rest_path[pos + 1..]); } None => { @@ -653,11 +676,18 @@ impl dyn IndexNode { } let inode = result.find(&name)?; + let file_type = inode.metadata()?.file_type; + // 如果已经是路径的最后一个部分，并且不希望跟随最后的符号链接 + if rest_path.is_empty() && !follow_final_symlink && file_type == FileType::SymLink { + // 返回符号链接本身 + return Ok(inode); + } - // 处理符号链接的问题 - if inode.metadata()?.file_type == FileType::SymLink && max_follow_times > 0 { + // 跟随符号链接跳转 + if file_type == FileType::SymLink && max_follow_times > 0 { let mut content = [0u8; 256]; // 读取符号链接 + let len = inode.read_at( 0, 256, @@ -667,12 +697,16 @@ impl dyn IndexNode { // 将读到的数据转换为utf8字符串（先转为str，再转为String） let link_path = String::from( - ::core::str::from_utf8(&content[..len]).map_err(|_| SystemError::ENOTDIR)?, + ::core::str::from_utf8(&content[..len]).map_err(|_| SystemError::EINVAL)?, ); - let new_path = link_path + "/" + &rest_path; + // 继续查找符号链接 - return result.lookup_follow_symlink(&new_path, max_follow_times - 1); + return result.lookup_follow_symlink2( + &new_path, + max_follow_times - 1, + follow_final_symlink, + ); } else { result = inode; } diff --git a/kernel/src/filesystem/vfs/open.rs b/kernel/src/filesystem/vfs/open.rs index 14f31fa10..108a114a7 100644 --- a/kernel/src/filesystem/vfs/open.rs +++ b/kernel/src/filesystem/vfs/open.rs @@ -9,12 +9,15 @@ use super::{ utils::{rsplit_path, user_path_at}, FileType, IndexNode, MAX_PATHLEN, ROOT_INODE, VFS_MAX_FOLLOW_SYMLINK_TIMES, }; -use crate::filesystem::vfs::syscall::UtimensFlags; -use crate::time::{syscall::PosixTimeval, PosixTimeSpec}; use crate::{ driver::base::block::SeekFrom, process::ProcessManager, syscall::user_access::check_and_clone_cstr, }; +use crate::{filesystem::vfs::syscall::UtimensFlags, process::cred::Kgid}; +use crate::{ + process::cred::GroupInfo, + time::{syscall::PosixTimeval, PosixTimeSpec}, +}; use alloc::string::String; pub(super) fn do_faccessat( @@ -64,6 +67,88 @@ pub fn do_fchmodat(dirfd: i32, path: *const u8, _mode: ModeType) -> Result Result { + // 检查flag是否合法 + if flag.contains(!(AtFlags::AT_SYMLINK_NOFOLLOW | AtFlags::AT_EMPTY_PATH)) { + return Err(SystemError::EINVAL); + } + + let follow_symlink = flag.contains(!AtFlags::AT_SYMLINK_NOFOLLOW); + let (inode, path) = user_path_at(&ProcessManager::current_pcb(), dirfd, path)?; + + // 如果找不到文件，则返回错误码ENOENT + let inode = if follow_symlink { + inode.lookup_follow_symlink2(path.as_str(), VFS_MAX_FOLLOW_SYMLINK_TIMES, false) + } else { + inode.lookup(path.as_str()) + }; + + if inode.is_err() { + let errno = inode.clone().unwrap_err(); + // 文件不存在 + if errno == SystemError::ENOENT { + return Err(SystemError::ENOENT); + } + } + + let inode = inode.unwrap(); + + return chown_common(inode, uid, gid); +} + +fn chown_common(inode: Arc, uid: usize, gid: usize) -> Result { + let mut meta = inode.metadata()?; + let cred = ProcessManager::current_pcb().cred(); + let current_uid = cred.uid.data(); + let current_gid = cred.gid.data(); + let mut group_info = GroupInfo::default(); + if let Some(info) = cred.group_info.as_ref() { + group_info = info.clone(); + } + + // 检查权限 + match current_uid { + 0 => { + meta.uid = uid; + meta.gid = gid; + } + _ => { + // 非文件所有者不能更改信息，且不能更改uid + if current_uid != meta.uid || uid != meta.uid { + return Err(SystemError::EPERM); + } + if gid != current_gid && !group_info.gids.contains(&Kgid::from(gid)) { + return Err(SystemError::EPERM); + } + meta.gid = gid; + } + } + + meta.mode.remove(ModeType::S_ISUID | ModeType::S_ISGID); + inode.set_metadata(&meta)?; + + return Ok(0); +} + +pub fn ksys_fchown(fd: i32, uid: usize, gid: usize) -> Result { + let fd_table = &ProcessManager::current_pcb().fd_table(); + let fd_table = fd_table.read(); + + let inode = fd_table.get_file_by_fd(fd).unwrap().inode(); + + let result = chown_common(inode, uid, gid); + + drop(fd_table); + + return result; +} + pub(super) fn do_sys_open( dfd: i32, path: &str, diff --git a/kernel/src/filesystem/vfs/syscall.rs b/kernel/src/filesystem/vfs/syscall.rs index 5d256bf03..ef8220ef7 100644 --- a/kernel/src/filesystem/vfs/syscall.rs +++ b/kernel/src/filesystem/vfs/syscall.rs @@ -25,7 +25,9 @@ use super::{ core::{do_mkdir_at, do_remove_dir, do_unlink_at}, fcntl::{AtFlags, FcntlCommand, FD_CLOEXEC}, file::{File, FileMode}, - open::{do_faccessat, do_fchmodat, do_sys_open, do_utimensat, do_utimes}, + open::{ + do_faccessat, do_fchmodat, do_fchownat, do_sys_open, do_utimensat, do_utimes, ksys_fchown, + }, utils::{rsplit_path, user_path_at}, Dirent, FileType, IndexNode, SuperBlock, FSMAKER, MAX_PATHLEN, ROOT_INODE, VFS_MAX_FOLLOW_SYMLINK_TIMES, @@ -1638,6 +1640,52 @@ impl Syscall { warn!("fchmod not fully implemented"); return Ok(0); } + + pub fn chown(pathname: *const u8, uid: usize, gid: usize) -> Result { + let pathname = user_access::check_and_clone_cstr(pathname, Some(MAX_PATHLEN))? + .into_string() + .map_err(|_| SystemError::EINVAL)?; + return do_fchownat( + AtFlags::AT_FDCWD.bits(), + &pathname, + uid, + gid, + AtFlags::AT_STATX_SYNC_AS_STAT, + ); + } + + pub fn lchown(pathname: *const u8, uid: usize, gid: usize) -> Result { + let pathname = user_access::check_and_clone_cstr(pathname, Some(MAX_PATHLEN))? + .into_string() + .map_err(|_| SystemError::EINVAL)?; + return do_fchownat( + AtFlags::AT_FDCWD.bits(), + &pathname, + uid, + gid, + AtFlags::AT_SYMLINK_NOFOLLOW, + ); + } + + pub fn fchownat( + dirfd: i32, + pathname: *const u8, + uid: usize, + gid: usize, + flags: i32, + ) -> Result { + let pathname = user_access::check_and_clone_cstr(pathname, Some(MAX_PATHLEN))? + .into_string() + .map_err(|_| SystemError::EINVAL)?; + let pathname = pathname.as_str().trim(); + let flags = AtFlags::from_bits_truncate(flags); + return do_fchownat(dirfd, pathname, uid, gid, flags); + } + + pub fn fchown(fd: i32, uid: usize, gid: usize) -> Result { + return ksys_fchown(fd, uid, gid); + } + /// #挂载文件系统 /// /// 用于挂载文件系统,目前仅支持ramfs挂载 diff --git a/kernel/src/process/cred.rs b/kernel/src/process/cred.rs index 952cfbfd5..6a6864c78 100644 --- a/kernel/src/process/cred.rs +++ b/kernel/src/process/cred.rs @@ -164,7 +164,7 @@ impl Cred { } } -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq, Eq, Default)] pub struct GroupInfo { pub gids: Vec, } diff --git a/kernel/src/syscall/mod.rs b/kernel/src/syscall/mod.rs index 6e7e9f7be..74f90610a 100644 --- a/kernel/src/syscall/mod.rs +++ b/kernel/src/syscall/mod.rs @@ -997,8 +997,32 @@ impl Syscall { } SYS_FCHOWN => { - warn!("SYS_FCHOWN has not yet been implemented"); - Ok(0) + let dirfd = args[0] as i32; + let uid = args[1]; + let gid = args[2]; + Self::fchown(dirfd, uid, gid) + } + #[cfg(target_arch = "x86_64")] + SYS_CHOWN => { + let pathname = args[0] as *const u8; + let uid = args[1]; + let gid = args[2]; + Self::chown(pathname, uid, gid) + } + #[cfg(target_arch = "x86_64")] + SYS_LCHOWN => { + let pathname = args[0] as *const u8; + let uid = args[1]; + let gid = args[2]; + Self::lchown(pathname, uid, gid) + } + SYS_FCHOWNAT => { + let dirfd = args[0] as i32; + let pathname = args[1] as *const u8; + let uid = args[2]; + let gid = args[3]; + let flag = args[4] as i32; + Self::fchownat(dirfd, pathname, uid, gid, flag) } SYS_FSYNC => { diff --git a/user/apps/test-chown/.gitignore b/user/apps/test-chown/.gitignore new file mode 100644 index 000000000..3e6e158be --- /dev/null +++ b/user/apps/test-chown/.gitignore @@ -0,0 +1,4 @@ +/target +Cargo.lock +testfile.txt +/install/ \ No newline at end of file diff --git a/user/apps/test-chown/Cargo.toml b/user/apps/test-chown/Cargo.toml new file mode 100644 index 000000000..5966be309 --- /dev/null +++ b/user/apps/test-chown/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "test-chown" +version = "0.1.0" +edition = "2021" +description = "测试chown系列系统调用" +authors = [ "sparkzky " ] + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +libc = "0.2" +errno = "0.3.9" +nix = "0.23" \ No newline at end of file diff --git a/user/apps/test-chown/Makefile b/user/apps/test-chown/Makefile new file mode 100644 index 000000000..7522ea16c --- /dev/null +++ b/user/apps/test-chown/Makefile @@ -0,0 +1,56 @@ +TOOLCHAIN= +RUSTFLAGS= + +ifdef DADK_CURRENT_BUILD_DIR +# 如果是在dadk中编译，那么安装到dadk的安装目录中 + INSTALL_DIR = $(DADK_CURRENT_BUILD_DIR) +else +# 如果是在本地编译，那么安装到当前目录下的install目录中 + INSTALL_DIR = ./install +endif + +ifeq ($(ARCH), x86_64) + export RUST_TARGET=x86_64-unknown-linux-musl +else ifeq ($(ARCH), riscv64) + export RUST_TARGET=riscv64gc-unknown-linux-gnu +else +# 默认为x86_86，用于本地编译 + export RUST_TARGET=x86_64-unknown-linux-musl +endif + +run: + RUSTFLAGS=$(RUSTFLAGS) cargo $(TOOLCHAIN) run --target $(RUST_TARGET) + +build: + RUSTFLAGS=$(RUSTFLAGS) cargo $(TOOLCHAIN) build --target $(RUST_TARGET) + +clean: + RUSTFLAGS=$(RUSTFLAGS) cargo $(TOOLCHAIN) clean --target $(RUST_TARGET) + +test: + RUSTFLAGS=$(RUSTFLAGS) cargo $(TOOLCHAIN) test --target $(RUST_TARGET) + +doc: + RUSTFLAGS=$(RUSTFLAGS) cargo $(TOOLCHAIN) doc --target $(RUST_TARGET) + +fmt: + RUSTFLAGS=$(RUSTFLAGS) cargo $(TOOLCHAIN) fmt + +fmt-check: + RUSTFLAGS=$(RUSTFLAGS) cargo $(TOOLCHAIN) fmt --check + +run-release: + RUSTFLAGS=$(RUSTFLAGS) cargo $(TOOLCHAIN) run --target $(RUST_TARGET) --release + +build-release: + RUSTFLAGS=$(RUSTFLAGS) cargo $(TOOLCHAIN) build --target $(RUST_TARGET) --release + +clean-release: + RUSTFLAGS=$(RUSTFLAGS) cargo $(TOOLCHAIN) clean --target $(RUST_TARGET) --release + +test-release: + RUSTFLAGS=$(RUSTFLAGS) cargo $(TOOLCHAIN) test --target $(RUST_TARGET) --release + +.PHONY: install +install: + RUSTFLAGS=$(RUSTFLAGS) cargo $(TOOLCHAIN) install --target $(RUST_TARGET) --path . --no-track --root $(INSTALL_DIR) --force diff --git a/user/apps/test-chown/README.md b/user/apps/test-chown/README.md new file mode 100644 index 000000000..1a4585a42 --- /dev/null +++ b/user/apps/test-chown/README.md @@ -0,0 +1,8 @@ +# 一个简单的用于测试chown系列系统调用的程序 + +### 由于symlink系统调用还未实现，目前只测试chown和fchown + +### 测试前需要手动添加nogroup用户组和nobody用户（程序里加不了） +```groupadd -g 65534 nogroup +useradd -d /nonexistent -g 65534 -u 65534 -s /usr/local/bin/false nobody +``` diff --git a/user/apps/test-chown/src/main.rs b/user/apps/test-chown/src/main.rs new file mode 100644 index 000000000..b2de5ef0e --- /dev/null +++ b/user/apps/test-chown/src/main.rs @@ -0,0 +1,160 @@ +use core::ffi::{c_char, c_void}; +use libc::{ + chown, fchown, fchownat, getgrnam, getpwnam, gid_t, lchown, mount, uid_t, umount, AT_FDCWD, + AT_SYMLINK_NOFOLLOW, +}; +use nix::errno::Errno; +use std::{ + ffi::CString, + fs::{self, metadata, File}, + io::{self, Error, Write}, + os::unix::{ + fs::{MetadataExt, PermissionsExt}, + io::AsRawFd, + }, + path::Path, +}; + +fn print_file_owner_group(filename: &str) -> Result<(), Error> { + let metadata = std::fs::metadata(filename)?; + let uid = metadata.uid(); + let gid = metadata.gid(); + + // 确保 UID 和 GID 打印正确 + assert!(uid > 0, "UID should be greater than 0"); + assert!(gid > 0, "GID should be greater than 0"); + + Ok(()) +} + +fn test_fchownat(filename: &str, new_uid: uid_t, new_gid: gid_t, flags: i32) -> Result<(), Error> { + let c_filename = CString::new(filename)?; + let result = unsafe { fchownat(AT_FDCWD, c_filename.as_ptr(), new_uid, new_gid, flags) }; + + // 确保 fchownat 成功 + assert!(result != -1, "fchownat failed"); + + print_file_owner_group(filename)?; + Ok(()) +} + +fn test_chown(filename: &str, new_uid: uid_t, new_gid: gid_t) -> Result<(), Error> { + let c_filename = CString::new(filename)?; + let result = unsafe { chown(c_filename.as_ptr(), new_uid, new_gid) }; + + // 确保 chown 成功 + assert!(result != -1, "chown failed"); + + print_file_owner_group(filename)?; + Ok(()) +} + +fn test_fchown(fd: i32, new_uid: uid_t, new_gid: gid_t) -> Result<(), Error> { + let result = unsafe { fchown(fd, new_uid, new_gid) }; + + // 确保 fchown 成功 + assert!(result != -1, "fchown failed"); + + Ok(()) +} + +fn test_lchown(symlink_name: &str, new_uid: uid_t, new_gid: gid_t) -> Result<(), Error> { + let c_symlink = CString::new(symlink_name)?; + let result = unsafe { lchown(c_symlink.as_ptr(), new_uid, new_gid) }; + + // 确保 lchown 成功 + assert!(result != -1, "lchown failed"); + + print_file_owner_group(symlink_name)?; + Ok(()) +} + +fn main() -> Result<(), Error> { + mount_test_ramfs(); + + let filename = "/mnt/myramfs/testfile.txt"; + let symlink_name = "/mnt/myramfs/testsymlink"; + let new_owner = "nobody"; // 替换为你测试系统中的有效用户名 + let new_group = "nogroup"; // 替换为你测试系统中的有效组名 + + // 获取新的 UID 和 GID + let pw = unsafe { getpwnam(CString::new(new_owner)?.as_ptr()) }; + let gr = unsafe { getgrnam(CString::new(new_group)?.as_ptr()) }; + + assert!(!pw.is_null(), "Invalid user name"); + assert!(!gr.is_null(), "Invalid group name"); + + let new_uid = unsafe { (*pw).pw_uid }; + let new_gid = unsafe { (*gr).gr_gid }; + + // 创建测试文件 + let mut file = File::create(filename)?; + println!("Created test file: {}", filename); + writeln!(file, "This is a test file for chown system call")?; + + // 创建符号链接 + std::os::unix::fs::symlink(filename, symlink_name)?; + println!("Created symlink: {}", symlink_name); + + // 打开文件以测试 fchown + let fd = file.as_raw_fd(); + + // 测试 chown + test_chown(filename, new_uid, new_gid)?; + + // 测试 fchown + test_fchown(fd, new_uid, new_gid)?; + + // 测试 lchown + test_lchown(symlink_name, new_uid, new_gid)?; + + // 测试 fchownat，带 AT_SYMLINK_NOFOLLOW 标志（不会跟随符号链接） + test_fchownat(symlink_name, new_uid, new_gid, AT_SYMLINK_NOFOLLOW)?; + + // 清理测试文件 + std::fs::remove_file(filename)?; + + umount_test_ramfs(); + + println!("All tests passed!"); + + Ok(()) +} + +fn mount_test_ramfs() { + let path = Path::new("mnt/myramfs"); + let dir = fs::create_dir_all(path); + assert!(dir.is_ok(), "mkdir /mnt/myramfs failed"); + + let source = b"\0".as_ptr() as *const c_char; + let target = b"/mnt/myramfs\0".as_ptr() as *const c_char; + let fstype = b"ramfs\0".as_ptr() as *const c_char; + // let flags = MS_BIND; + let flags = 0; + let data = std::ptr::null() as *const c_void; + let result = unsafe { mount(source, target, fstype, flags, data) }; + + assert_eq!( + result, + 0, + "Mount myramfs failed, errno: {}", + Errno::last().desc() + ); + println!("Mount myramfs for test success!"); +} + +fn umount_test_ramfs() { + let path = b"/mnt/myramfs\0".as_ptr() as *const c_char; + let result = unsafe { umount(path) }; + if result != 0 { + let err = Errno::last(); + println!("Errno: {}", err); + println!("Infomation: {}", err.desc()); + } else { + // 删除mnt/myramfs + let path = Path::new("mnt/myramfs"); + let _ = fs::remove_dir(path); + } + assert_eq!(result, 0, "Umount myramfs failed"); + println!("Umount myramfs for test success!"); +} diff --git a/user/dadk/config/test_chown_0_1_0.dadk b/user/dadk/config/test_chown_0_1_0.dadk new file mode 100644 index 000000000..5da86f07d --- /dev/null +++ b/user/dadk/config/test_chown_0_1_0.dadk @@ -0,0 +1,29 @@ +{ + "name": "test-chown", + "version": "0.1.0", + "description": "chown系列系统调用", + "rust_target": "x86_64-unknown-dragonos", + "task_type": { + "BuildFromSource": { + "Local": { + "path": "apps/test-chown" + } + } + }, + "depends": [], + "build": { + "build_command": "make install" + }, + "install": { + "in_dragonos_path": "/" + }, + "clean": { + "clean_command": "make clean" + }, + "envs": [], + "build_once": false, + "install_once": false, + "target_arch": [ + "x86_64" + ] +} \ No newline at end of file From 84c528f53d91118bfc79e6d706b5b2de31e89915 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=BB=84=E9=93=AD=E6=B6=9B?= <114841534+1037827920@users.noreply.github.com> Date: Tue, 29 Oct 2024 00:26:16 +0800 Subject: [PATCH 15/30] =?UTF-8?q?fix(driver/virtio):=20=E4=BF=AE=E6=94=B9p?= =?UTF-8?q?ci=20transport=E4=B8=AD=E6=96=AD=E5=88=9D=E5=A7=8B=E5=8C=96?= =?UTF-8?q?=E7=9A=84=E4=BD=8D=E7=BD=AE=20(#1018)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(driver/virtio): 修改pci transport中断初始化的位置 --- kernel/src/driver/block/virtio_blk.rs | 8 +++- kernel/src/driver/net/virtio_net.rs | 6 +++ kernel/src/driver/virtio/transport.rs | 53 +++++++++++++++++++---- kernel/src/driver/virtio/transport_pci.rs | 33 +++++--------- kernel/src/net/net_core.rs | 14 +++++- 5 files changed, 83 insertions(+), 31 deletions(-) diff --git a/kernel/src/driver/block/virtio_blk.rs b/kernel/src/driver/block/virtio_blk.rs index 136544247..6240263b5 100644 --- a/kernel/src/driver/block/virtio_blk.rs +++ b/kernel/src/driver/block/virtio_blk.rs @@ -163,8 +163,14 @@ unsafe impl Sync for VirtIOBlkDevice {} impl VirtIOBlkDevice { pub fn new(transport: VirtIOTransport, dev_id: Arc) -> Option> { + // 设置中断 + if let Err(err) = transport.setup_irq(dev_id.clone()) { + error!("VirtIOBlkDevice '{dev_id:?}' setup_irq failed: {:?}", err); + return None; + } + let devname = virtioblk_manager().alloc_id()?; - let irq = transport.irq().map(|irq| IrqNumber::new(irq.data())); + let irq = Some(transport.irq()); let device_inner = VirtIOBlk::::new(transport); if let Err(e) = device_inner { error!("VirtIOBlkDevice '{dev_id:?}' create failed: {:?}", e); diff --git a/kernel/src/driver/net/virtio_net.rs b/kernel/src/driver/net/virtio_net.rs index 0bf2e3bff..521f2752b 100644 --- a/kernel/src/driver/net/virtio_net.rs +++ b/kernel/src/driver/net/virtio_net.rs @@ -91,6 +91,12 @@ impl Debug for InnerVirtIONetDevice { impl VirtIONetDevice { pub fn new(transport: VirtIOTransport, dev_id: Arc) -> Option> { + // 设置中断 + if let Err(err) = transport.setup_irq(dev_id.clone()) { + error!("VirtIONetDevice '{dev_id:?}' setup_irq failed: {:?}", err); + return None; + } + let driver_net: VirtIONet = match VirtIONet::::new(transport, 4096) { Ok(net) => net, diff --git a/kernel/src/driver/virtio/transport.rs b/kernel/src/driver/virtio/transport.rs index b18194035..9dab04795 100644 --- a/kernel/src/driver/virtio/transport.rs +++ b/kernel/src/driver/virtio/transport.rs @@ -1,8 +1,21 @@ -use virtio_drivers::transport::Transport; +use alloc::{string::ToString, sync::Arc}; -use crate::exception::HardwareIrqNumber; +use virtio_drivers::transport::Transport; -use super::{transport_mmio::VirtIOMmioTransport, transport_pci::PciTransport}; +use crate::{ + driver::{ + base::device::DeviceId, + pci::{ + pci::{PciDeviceStructure, PciError}, + pci_irq::{IrqCommonMsg, IrqSpecificMsg, PciInterrupt, PciIrqError, PciIrqMsg, IRQ}, + }, + }, + exception::IrqNumber, +}; + +use super::{ + irq::DefaultVirtioIrqHandler, transport_mmio::VirtIOMmioTransport, transport_pci::PciTransport, +}; pub enum VirtIOTransport { Pci(PciTransport), @@ -10,11 +23,35 @@ pub enum VirtIOTransport { } impl VirtIOTransport { - pub fn irq(&self) -> Option { - match self { - VirtIOTransport::Mmio(transport) => Some(transport.irq()), - _ => None, - } + pub fn irq(&self) -> IrqNumber { + match self { + VirtIOTransport::Pci(transport) => transport.irq(), + VirtIOTransport::Mmio(transport) => IrqNumber::new(transport.irq().data()), + } + } + + /// 设置中断 + pub fn setup_irq(&self, dev_id: Arc) -> Result<(), PciError> { + if let VirtIOTransport::Pci(transport) = self { + let mut pci_device_guard = transport.pci_device(); + let standard_device = pci_device_guard.as_standard_device_mut().unwrap(); + standard_device + .irq_init(IRQ::PCI_IRQ_MSIX | IRQ::PCI_IRQ_MSI) + .ok_or(PciError::PciIrqError(PciIrqError::IrqNotInited))?; + // 中断相关信息 + let msg = PciIrqMsg { + irq_common_message: IrqCommonMsg::init_from( + 0, + "Virtio_IRQ".to_string(), + &DefaultVirtioIrqHandler, + dev_id, + ), + irq_specific_message: IrqSpecificMsg::msi_default(), + }; + standard_device.irq_install(msg)?; + standard_device.irq_enable(true)?; + } + return Ok(()); } } diff --git a/kernel/src/driver/virtio/transport_pci.rs b/kernel/src/driver/virtio/transport_pci.rs index 07b687ba1..5ecdfed26 100644 --- a/kernel/src/driver/virtio/transport_pci.rs +++ b/kernel/src/driver/virtio/transport_pci.rs @@ -6,17 +6,16 @@ use crate::driver::pci::pci::{ PciStandardDeviceBar, PCI_CAP_ID_VNDR, }; -use crate::driver::pci::pci_irq::{IrqCommonMsg, IrqSpecificMsg, PciInterrupt, PciIrqMsg, IRQ}; use crate::driver::pci::root::pci_root_0; use crate::exception::IrqNumber; +use crate::libs::spinlock::{SpinLock, SpinLockGuard}; use crate::libs::volatile::{ volread, volwrite, ReadOnly, Volatile, VolatileReadable, VolatileWritable, WriteOnly, }; use crate::mm::VirtAddr; -use alloc::string::ToString; use alloc::sync::Arc; use core::{ fmt::{self, Display, Formatter}, @@ -28,7 +27,6 @@ use virtio_drivers::{ Error, Hal, PhysAddr, }; -use super::irq::DefaultVirtioIrqHandler; use super::VIRTIO_VENDOR_ID; /// The offset to add to a VirtIO device ID to get the corresponding PCI device ID. @@ -104,6 +102,7 @@ pub struct PciTransport { config_space: Option>, irq: IrqNumber, dev_id: Arc, + device: Arc>, } impl PciTransport { @@ -140,21 +139,7 @@ impl PciTransport { // 目前缺少对PCI设备中断号的统一管理，所以这里需要指定一个中断号。不能与其他中断重复 let irq_vector = standard_device.irq_vector_mut().unwrap(); irq_vector.push(irq); - standard_device - .irq_init(IRQ::PCI_IRQ_MSIX | IRQ::PCI_IRQ_MSI) - .ok_or(VirtioPciError::UnableToInitIrq)?; - // 中断相关信息 - let msg = PciIrqMsg { - irq_common_message: IrqCommonMsg::init_from( - 0, - "Virtio_IRQ".to_string(), - &DefaultVirtioIrqHandler, - dev_id.clone(), - ), - irq_specific_message: IrqSpecificMsg::msi_default(), - }; - standard_device.irq_install(msg)?; - standard_device.irq_enable(true)?; + //device_capability为迭代器，遍历其相当于遍历所有的cap空间 for capability in device.capabilities().unwrap() { if capability.id != PCI_CAP_ID_VNDR { @@ -236,8 +221,17 @@ impl PciTransport { config_space, irq, dev_id, + device: Arc::new(SpinLock::new(device.clone())), }) } + + pub fn pci_device(&self) -> SpinLockGuard { + self.device.lock() + } + + pub fn irq(&self) -> IrqNumber { + self.irq + } } impl Transport for PciTransport { @@ -446,8 +440,6 @@ pub enum VirtioPciError { /// `VIRTIO_PCI_CAP_NOTIFY_CFG` capability has a `notify_off_multiplier` that is not a multiple /// of 2. InvalidNotifyOffMultiplier(u32), - /// Unable to find capability such as MSIX or MSI. - UnableToInitIrq, /// No valid `VIRTIO_PCI_CAP_ISR_CFG` capability was found. MissingIsrConfig, /// An IO BAR was provided rather than a memory BAR. @@ -477,7 +469,6 @@ impl Display for VirtioPciError { "PCI device vender ID {:#06x} was not the VirtIO vendor ID {:#06x}.", vendor_id, VIRTIO_VENDOR_ID ), - Self::UnableToInitIrq => write!(f, "Unable to find capability such as MSIX or MSI."), Self::MissingCommonConfig => write!( f, "No valid `VIRTIO_PCI_CAP_COMMON_CFG` capability was found." diff --git a/kernel/src/net/net_core.rs b/kernel/src/net/net_core.rs index 9de8054eb..b90a35aa6 100644 --- a/kernel/src/net/net_core.rs +++ b/kernel/src/net/net_core.rs @@ -7,7 +7,11 @@ use crate::{ driver::net::{NetDevice, Operstate}, libs::rwlock::RwLockReadGuard, net::{socket::SocketPollMethod, NET_DEVICES}, - time::timer::{next_n_ms_timer_jiffies, Timer, TimerFunction}, + time::{ + sleep::nanosleep, + timer::{next_n_ms_timer_jiffies, Timer, TimerFunction}, + PosixTimeSpec, + }, }; use super::{ @@ -118,6 +122,14 @@ fn dhcp_query() -> Result<(), SystemError> { .remove_default_ipv4_route(); } } + // 在睡眠前释放锁 + drop(binding); + + let sleep_time = PosixTimeSpec { + tv_sec: 5, + tv_nsec: 0, + }; + let _ = nanosleep(sleep_time)?; } return Err(SystemError::ETIMEDOUT); From f5b2038871d3441e1c7f32439ff422957e7ab828 Mon Sep 17 00:00:00 2001 From: codeironman <1553389239@qq.com> Date: Thu, 31 Oct 2024 00:50:34 +0800 Subject: [PATCH 16/30] ospp project (feature) add namespace overlayfs cgroup (#949) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## 开发进展： ## namespace - pid_namespace 基本实现，基于pid_struct等数据结构实现隔离 - mnt_namespace 基本实现，挂载点的隔离通过不同的挂载树来实现 - usernamespace 作为支持性的namespace，目前受限实现全局静态 ## overlayfs - 实现若干个文件系统的叠加，在mount中传入多个路径作为多个fs的mount路径以及最后merge层的fs路径 - copy-up机制的，除最上层外其他层为只读层，满足写时拷贝，需要修改的时候copy到上层修改 - whiteout特殊文件，用于标记在下层需要被删除的文件用来掩盖需要删除的文件 ## cgroups - 目前cgroups还处于框架阶段，之后具体实现具体的内存、CPU等子系统 --- docs/index.rst | 2 + docs/kernel/container/index.rst | 13 + docs/kernel/container/namespaces/index.rst | 14 + .../container/namespaces/mnt_namespace.md | 19 + .../container/namespaces/pid_namespace.md | 21 + docs/kernel/filesystem/index.rst | 1 + docs/kernel/filesystem/unionfs/index.rst | 10 + docs/kernel/filesystem/unionfs/overlayfs.md | 26 ++ kernel/crates/ida/src/lib.rs | 5 + kernel/src/arch/riscv64/process/mod.rs | 2 +- kernel/src/arch/x86_64/process/mod.rs | 2 +- kernel/src/cgroup/mem_cgroup.rs | 6 + kernel/src/cgroup/mod.rs | 48 ++ kernel/src/filesystem/mod.rs | 1 + kernel/src/filesystem/overlayfs/copy_up.rs | 41 ++ kernel/src/filesystem/overlayfs/entry.rs | 32 ++ kernel/src/filesystem/overlayfs/mod.rs | 432 ++++++++++++++++++ kernel/src/filesystem/ramfs/mod.rs | 69 +-- kernel/src/filesystem/vfs/mod.rs | 27 +- kernel/src/filesystem/vfs/syscall.rs | 7 +- kernel/src/init/initial_kthread.rs | 3 + kernel/src/lib.rs | 5 +- kernel/src/libs/ida/src/lib.rs | 55 +++ kernel/src/namespaces/mnt_namespace.rs | 206 +++++++++ kernel/src/namespaces/mod.rs | 92 ++++ kernel/src/namespaces/namespace.rs | 119 +++++ kernel/src/namespaces/pid_namespace.rs | 273 +++++++++++ kernel/src/namespaces/syscall.rs | 50 ++ kernel/src/namespaces/ucount.rs | 187 ++++++++ kernel/src/namespaces/user_namespace.rs | 135 ++++++ kernel/src/process/fork.rs | 52 ++- kernel/src/process/mod.rs | 39 +- kernel/src/process/syscall.rs | 8 +- kernel/src/syscall/mod.rs | 6 +- user/apps/test_namespace/Cargo.toml | 7 + user/apps/test_namespace/Makefile | 56 +++ user/apps/test_namespace/makefile.toml | 63 +++ user/apps/test_namespace/src/main.rs | 38 ++ user/apps/test_overlayfs/.gitignore | 1 + user/apps/test_overlayfs/Makefile | 20 + user/apps/test_overlayfs/main.c | 92 ++++ user/dadk/config/test_namespace.dadk | 25 + user/dadk/config/test_overlayfs.dadk | 25 + 43 files changed, 2279 insertions(+), 56 deletions(-) create mode 100644 docs/kernel/container/index.rst create mode 100644 docs/kernel/container/namespaces/index.rst create mode 100644 docs/kernel/container/namespaces/mnt_namespace.md create mode 100644 docs/kernel/container/namespaces/pid_namespace.md create mode 100644 docs/kernel/filesystem/unionfs/index.rst create mode 100644 docs/kernel/filesystem/unionfs/overlayfs.md create mode 100644 kernel/src/cgroup/mem_cgroup.rs create mode 100644 kernel/src/cgroup/mod.rs create mode 100644 kernel/src/filesystem/overlayfs/copy_up.rs create mode 100644 kernel/src/filesystem/overlayfs/entry.rs create mode 100644 kernel/src/filesystem/overlayfs/mod.rs create mode 100644 kernel/src/libs/ida/src/lib.rs create mode 100644 kernel/src/namespaces/mnt_namespace.rs create mode 100644 kernel/src/namespaces/mod.rs create mode 100644 kernel/src/namespaces/namespace.rs create mode 100644 kernel/src/namespaces/pid_namespace.rs create mode 100644 kernel/src/namespaces/syscall.rs create mode 100644 kernel/src/namespaces/ucount.rs create mode 100644 kernel/src/namespaces/user_namespace.rs create mode 100644 user/apps/test_namespace/Cargo.toml create mode 100644 user/apps/test_namespace/Makefile create mode 100644 user/apps/test_namespace/makefile.toml create mode 100644 user/apps/test_namespace/src/main.rs create mode 100644 user/apps/test_overlayfs/.gitignore create mode 100644 user/apps/test_overlayfs/Makefile create mode 100644 user/apps/test_overlayfs/main.c create mode 100644 user/dadk/config/test_namespace.dadk create mode 100644 user/dadk/config/test_overlayfs.dadk diff --git a/docs/index.rst b/docs/index.rst index 5d2aa82b3..a0d9c869f 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -30,10 +30,12 @@ kernel/debug/index kernel/ktest/index kernel/cpu_arch/index + kernel/container/index kernel/libs/index kernel/trace/index + .. toctree:: :maxdepth: 1 :caption: 应用层 diff --git a/docs/kernel/container/index.rst b/docs/kernel/container/index.rst new file mode 100644 index 000000000..0a5709d3a --- /dev/null +++ b/docs/kernel/container/index.rst @@ -0,0 +1,13 @@ +==================================== +容器化 +==================================== + + 这里是DragonOS中，与容器化相关的说明文档。 + + 主要包括namespace，overlayfs和cgroup + +.. toctree:: + :maxdepth: 2 + + namespaces/index + filesystem/unionfs/index diff --git a/docs/kernel/container/namespaces/index.rst b/docs/kernel/container/namespaces/index.rst new file mode 100644 index 000000000..2cfe02100 --- /dev/null +++ b/docs/kernel/container/namespaces/index.rst @@ -0,0 +1,14 @@ +==================================== +名称空间 +==================================== + +DragonOS的namespaces目前支持pid_namespace和mnt_namespace 预计之后会继续完善 +namespace是容器化实现过程中的重要组成部分 + +由于目前os是单用户，user_namespace为全局静态 + +.. toctree:: + :maxdepth: 1 + + pid_namespace + mnt_namespace diff --git a/docs/kernel/container/namespaces/mnt_namespace.md b/docs/kernel/container/namespaces/mnt_namespace.md new file mode 100644 index 000000000..dd828adfb --- /dev/null +++ b/docs/kernel/container/namespaces/mnt_namespace.md @@ -0,0 +1,19 @@ +# 挂载名称空间 + +## 底层架构 + +pcb -> nsproxy -> mnt_namespace + +每一个挂载文件系统都有自立独立的挂载点，表现在数据结构上是一个挂载的红黑树，每一个名称空间中挂载是独立的，所以文件系统的挂载和卸载不会影响别的 + +## 系统调用接口 + + +- clone + - CLONE_NEWNS用于创建一个新的 MNT 命名空间。提供独立的文件系统挂载点 +- unshare + - 使用 CLONE_NEWPID 标志调用 unshare() 后，后续创建的所有子进程都将在新的命名空间中运行。 +- setns + - 将进程加入到指定的名称空间 +- chroot + - 将当前进程的根目录更改为指定的路径，提供文件系统隔离。 \ No newline at end of file diff --git a/docs/kernel/container/namespaces/pid_namespace.md b/docs/kernel/container/namespaces/pid_namespace.md new file mode 100644 index 000000000..4e9210cc0 --- /dev/null +++ b/docs/kernel/container/namespaces/pid_namespace.md @@ -0,0 +1,21 @@ +# 进程名称空间 +:::{note} 本文作者：操丰毅 1553389239@qq.com + +2024年10月30日 ::: +pid_namespace 是内核中的一种名称空间，用于实现进程隔离，允许在不同的名称空间中运行的进程有独立的pid试图 + +## 底层架构 + +pcb -> nsproxy -> pid_namespace +- pid_namespace 内有独立的一套进程分配器，以及孤儿进程回收器，独立管理内部的pid +- 不同进程的详细信息都存放在proc文件系统中，里面的找到对应的pid号里面的信息都在pid中，记录的是pid_namespace中的信息 +- pid_namespace等限制由ucount来控制管理 + +## 系统调用接口 + +- clone + - CLONE_NEWPID用于创建一个新的 PID 命名空间。使用这个标志时，子进程将在新的 PID 命名空间内运行，进程 ID 从 1 开始。 +- unshare + - 使用 CLONE_NEWPID 标志调用 unshare() 后，后续创建的所有子进程都将在新的命名空间中运行。 +- getpid + - 在命名空间中调用 getpid() 会返回进程在当前 PID 命名空间中的进程 ID \ No newline at end of file diff --git a/docs/kernel/filesystem/index.rst b/docs/kernel/filesystem/index.rst index ff0c01b01..ef81b8ddb 100644 --- a/docs/kernel/filesystem/index.rst +++ b/docs/kernel/filesystem/index.rst @@ -13,4 +13,5 @@ todo: 由于文件系统模块重构，文档暂时不可用，预计在2023年4 vfs/index sysfs kernfs + unionfs/index diff --git a/docs/kernel/filesystem/unionfs/index.rst b/docs/kernel/filesystem/unionfs/index.rst new file mode 100644 index 000000000..141fb7a29 --- /dev/null +++ b/docs/kernel/filesystem/unionfs/index.rst @@ -0,0 +1,10 @@ +==================================== +联合文件系统 +==================================== +Union Filesystem： +OverlayFS 将多个文件系统（称为“层”）合并为一个逻辑文件系统，使用户看到一个统一的目录结构。 + +.. toctree:: + :maxdepth: 1 + + overlayfs diff --git a/docs/kernel/filesystem/unionfs/overlayfs.md b/docs/kernel/filesystem/unionfs/overlayfs.md new file mode 100644 index 000000000..33ae97bd4 --- /dev/null +++ b/docs/kernel/filesystem/unionfs/overlayfs.md @@ -0,0 +1,26 @@ +# overlayfs + +OverlayFs是目前使用最多的联合文件系统，原理简单方便使用，主要用于容器中 +在 Docker 中，OverlayFS 是默认的存储驱动之一。Docker 为每个容器创建一个独立的上层目录，而所有容器共享同一个下层镜像文件。这样的设计使得容器之间的资源共享更加高效，同时减少了存储需求。 +## 架构设计 +overlayfs主要有两个层，以及一个虚拟的合并层 +- Lower Layer（下层）：通常是 只读 文件系统。可以包含多层。 +- Upper Layer（上层）：为 可写层，所有的写操作都会在这一层上进行。 +- Merged Layer（合并层）：上层和下层的逻辑视图合并后，向用户呈现的最终文件系统。 + + +## 工作原理 +- 读取操作： + - OverlayFS 会优先从 Upper Layer 读取文件。如果文件不存在于上层，则读取 Lower Layer 中的内容。 +- 写入操作： + - 如果一个文件位于 Lower Layer 中，并尝试写入该文件，系统会将其 copy-up 到 Upper Layer 并在上层写入。如果文件已经存在于 Upper Layer，则直接在该层写入。 +- 删除操作： + - 当删除文件时，OverlayFS 会在上层创建一个标记为 whiteout 的条目，这会隐藏下层的文件。 + +## Copy-up +- 写时拷贝 +当一个文件从 下层 被修改时，它会被复制到 上层（称为 copy-up）。之后的所有修改都会发生在上层的文件副本上。 + + +## 实现逻辑 +通过构建ovlInode来实现indexnode这个trait来代表上层或者下层的inode，具体的有关文件文件夹的操作都在 \ No newline at end of file diff --git a/kernel/crates/ida/src/lib.rs b/kernel/crates/ida/src/lib.rs index 5d0d85a79..d9676bc92 100644 --- a/kernel/crates/ida/src/lib.rs +++ b/kernel/crates/ida/src/lib.rs @@ -140,6 +140,11 @@ impl IdAllocator { pub fn used(&self) -> usize { self.used } + + /// 返回最大id数 + pub fn get_max_id(&self) -> usize { + self.max_id + } } impl core::fmt::Debug for IdAllocator { diff --git a/kernel/src/arch/riscv64/process/mod.rs b/kernel/src/arch/riscv64/process/mod.rs index 89ec982d3..f7d4e2258 100644 --- a/kernel/src/arch/riscv64/process/mod.rs +++ b/kernel/src/arch/riscv64/process/mod.rs @@ -104,7 +104,7 @@ impl ProcessManager { pub fn copy_thread( current_pcb: &Arc, new_pcb: &Arc, - clone_args: KernelCloneArgs, + clone_args: &KernelCloneArgs, current_trapframe: &TrapFrame, ) -> Result<(), SystemError> { let clone_flags = clone_args.flags; diff --git a/kernel/src/arch/x86_64/process/mod.rs b/kernel/src/arch/x86_64/process/mod.rs index c4382cdc7..be2f53387 100644 --- a/kernel/src/arch/x86_64/process/mod.rs +++ b/kernel/src/arch/x86_64/process/mod.rs @@ -299,7 +299,7 @@ impl ProcessManager { pub fn copy_thread( current_pcb: &Arc, new_pcb: &Arc, - clone_args: KernelCloneArgs, + clone_args: &KernelCloneArgs, current_trapframe: &TrapFrame, ) -> Result<(), SystemError> { let clone_flags = clone_args.flags; diff --git a/kernel/src/cgroup/mem_cgroup.rs b/kernel/src/cgroup/mem_cgroup.rs new file mode 100644 index 000000000..1d1381929 --- /dev/null +++ b/kernel/src/cgroup/mem_cgroup.rs @@ -0,0 +1,6 @@ +use super::CgroupSubsysState; + +struct MemCgroup { + css: CgroupSubsysState, + id: u32, +} diff --git a/kernel/src/cgroup/mod.rs b/kernel/src/cgroup/mod.rs new file mode 100644 index 000000000..baed2f199 --- /dev/null +++ b/kernel/src/cgroup/mod.rs @@ -0,0 +1,48 @@ +#![allow(dead_code, unused_variables, unused_imports)] +pub mod mem_cgroup; + +use alloc::{collections::LinkedList, rc::Weak, sync::Arc, vec::Vec}; + +use alloc::boxed::Box; + +use crate::filesystem::vfs::IndexNode; + +pub struct Cgroup { + css: Weak, + /// 当前所在的深度 + level: u32, + /// 支持的最大深度 + max_depth: u32, + /// 可见后代数量 + nr_descendants: u32, + /// 正在死亡后代数量 + nr_dying_descendants: u32, + /// 允许的最大后代数量 + max_descendants: u32, + /// css_set的数量 + nr_populated_csets: u32, + /// 子group中有任务的记数 + nr_populated_domain_children: u32, + /// 线程子group中有任务的记数 + nr_populated_threaded_children: u32, + /// 活跃线程子cgroup数量 + nr_threaded_children: u32, + /// 关联cgroup的inode + kernfs_node: Box, +} + +/// 控制资源的统计信息 +pub struct CgroupSubsysState { + cgroup: Arc, + /// 兄弟节点 + sibling: LinkedList>, + /// 孩子节点 + children: LinkedList>, +} + +pub struct CgroupSubsys {} + +/// cgroup_sub_state 的集合 +pub struct CssSet { + subsys: Vec>, +} diff --git a/kernel/src/filesystem/mod.rs b/kernel/src/filesystem/mod.rs index 90dcc51bf..772d21f41 100644 --- a/kernel/src/filesystem/mod.rs +++ b/kernel/src/filesystem/mod.rs @@ -4,6 +4,7 @@ pub mod eventfd; pub mod fat; pub mod kernfs; pub mod mbr; +pub mod overlayfs; pub mod procfs; pub mod ramfs; pub mod sysfs; diff --git a/kernel/src/filesystem/overlayfs/copy_up.rs b/kernel/src/filesystem/overlayfs/copy_up.rs new file mode 100644 index 000000000..80b94aefa --- /dev/null +++ b/kernel/src/filesystem/overlayfs/copy_up.rs @@ -0,0 +1,41 @@ +use super::OvlInode; +use crate::{ + filesystem::vfs::{IndexNode, Metadata}, + libs::spinlock::SpinLock, +}; +use alloc::sync::Arc; +use system_error::SystemError; + +impl OvlInode { + pub fn copy_up(&self) -> Result<(), SystemError> { + let mut upper_inode = self.upper_inode.lock(); + if upper_inode.is_some() { + return Ok(()); + } + + let lower_inode = self.lower_inode.as_ref().ok_or(SystemError::ENOENT)?; + + let metadata = lower_inode.metadata()?; + let new_upper_inode = self.create_upper_inode(metadata.clone())?; + + let mut buffer = vec![0u8; metadata.size as usize]; + let lock = SpinLock::new(crate::filesystem::vfs::FilePrivateData::Unused); + lower_inode.read_at(0, metadata.size as usize, &mut buffer, lock.lock())?; + + new_upper_inode.write_at(0, metadata.size as usize, &buffer, lock.lock())?; + + *upper_inode = Some(new_upper_inode); + + Ok(()) + } + + fn create_upper_inode(&self, metadata: Metadata) -> Result, SystemError> { + let upper_inode = self.upper_inode.lock(); + let upper_root_inode = upper_inode + .as_ref() + .ok_or(SystemError::ENOSYS)? + .fs() + .root_inode(); + upper_root_inode.create_with_data(&self.dname()?.0, metadata.file_type, metadata.mode, 0) + } +} diff --git a/kernel/src/filesystem/overlayfs/entry.rs b/kernel/src/filesystem/overlayfs/entry.rs new file mode 100644 index 000000000..d6d69c87d --- /dev/null +++ b/kernel/src/filesystem/overlayfs/entry.rs @@ -0,0 +1,32 @@ +use alloc::sync::Arc; + +use alloc::vec::Vec; + +use crate::filesystem::vfs::IndexNode; + +use super::{OvlInode, OvlSuperBlock}; +#[derive(Debug)] +pub struct OvlEntry { + numlower: usize, // 下层数量 + lowerstack: Vec, +} + +impl OvlEntry { + pub fn new() -> Self { + Self { + numlower: 2, + lowerstack: Vec::new(), + } + } +} +#[derive(Debug)] +pub struct OvlPath { + layer: Arc, + inode: Arc, +} +#[derive(Debug)] +pub struct OvlLayer { + pub mnt: Arc, // 挂载点 + pub index: u32, // 0 是上层读写层，>0 是下层只读层 + pub fsid: u32, // 文件系统标识符 +} diff --git a/kernel/src/filesystem/overlayfs/mod.rs b/kernel/src/filesystem/overlayfs/mod.rs new file mode 100644 index 000000000..244217e32 --- /dev/null +++ b/kernel/src/filesystem/overlayfs/mod.rs @@ -0,0 +1,432 @@ +#![allow(dead_code, unused_variables, unused_imports)] +pub mod copy_up; +pub mod entry; + +use super::ramfs::{LockedRamFSInode, RamFSInode}; +use super::vfs::{self, FileSystem, FileType, FsInfo, IndexNode, Metadata, SuperBlock}; +use super::vfs::{FSMAKER, ROOT_INODE}; +use crate::driver::base::device::device_number::DeviceNumber; +use crate::driver::base::device::device_number::Major; +use crate::filesystem::vfs::{FileSystemMaker, FileSystemMakerData}; +use crate::libs::spinlock::SpinLock; +use alloc::string::String; +use alloc::sync::Arc; +use alloc::sync::Weak; +use alloc::vec::Vec; +use entry::{OvlEntry, OvlLayer}; +use system_error::SystemError; + +const WHITEOUT_MODE: u64 = 0o020000 | 0o600; // whiteout字符设备文件模式与权限 +const WHITEOUT_DEV: DeviceNumber = DeviceNumber::new(Major::UNNAMED_MAJOR, 0); // Whiteout 文件设备号 +const WHITEOUT_FLAG: u64 = 0x1; + +#[distributed_slice(FSMAKER)] +static OVERLAYFSMAKER: FileSystemMaker = FileSystemMaker::new( + "overlay", + &(OverlayFS::make_overlayfs + as fn( + Option<&dyn FileSystemMakerData>, + ) -> Result, SystemError>), +); +#[derive(Debug)] +pub struct OverlayMountData { + upper_dir: String, + lower_dirs: Vec, + work_dir: String, +} + +impl OverlayMountData { + pub fn from_row(raw_data: *const u8) -> Result { + if raw_data.is_null() { + return Err(SystemError::EINVAL); + } + let len = (0..) + .find(|&i| unsafe { raw_data.add(i).read() } == 0) + .ok_or(SystemError::EINVAL)?; + let slice = unsafe { core::slice::from_raw_parts(raw_data, len) }; + let raw_str = core::str::from_utf8(slice).map_err(|_| SystemError::EINVAL)?; + let mut data = OverlayMountData { + upper_dir: String::new(), + lower_dirs: Vec::new(), + work_dir: String::new(), + }; + + for pair in raw_str.split(',') { + let mut parts = pair.split('='); + let key = parts.next().ok_or(SystemError::EINVAL)?; + let value = parts.next().ok_or(SystemError::EINVAL)?; + + match key { + "upperdir" => data.upper_dir = value.into(), + "lowerdir" => data.lower_dirs = value.split(':').map(|s| s.into()).collect(), + "workdir" => data.work_dir = value.into(), + _ => return Err(SystemError::EINVAL), + } + } + Ok(data) + } +} +impl FileSystemMakerData for OverlayMountData { + fn as_any(&self) -> &dyn core::any::Any { + self + } +} +#[derive(Debug)] +pub struct OvlSuperBlock { + super_block: SuperBlock, + pseudo_dev: DeviceNumber, // 虚拟设备号 + is_lower: bool, +} + +#[derive(Debug)] +struct OverlayFS { + numlayer: usize, + numfs: u32, + numdatalayer: usize, + layers: Vec, // 第0层为读写层，后面是只读层 + workdir: Arc, + root_inode: Arc, +} + +#[derive(Debug)] +struct OvlInode { + redirect: String, // 重定向路径 + file_type: FileType, + flags: SpinLock, + upper_inode: SpinLock>>, // 读写层 + lower_inode: Option>, // 只读层 + oe: Arc, + fs: Weak, +} +impl OvlInode { + pub fn new( + redirect: String, + upper: Option>, + lower_inode: Option>, + ) -> Self { + Self { + redirect, + file_type: FileType::Dir, + flags: SpinLock::new(0), + upper_inode: SpinLock::new(upper), + lower_inode, + oe: Arc::new(OvlEntry::new()), + fs: Weak::default(), + } + } +} + +impl FileSystem for OverlayFS { + fn root_inode(&self) -> Arc { + self.root_inode.clone() + } + + fn info(&self) -> vfs::FsInfo { + FsInfo { + blk_dev_id: 0, + max_name_len: 255, + } + } + + fn as_any_ref(&self) -> &dyn core::any::Any { + self + } + + fn name(&self) -> &str { + "overlayfs" + } + + fn super_block(&self) -> SuperBlock { + todo!() + } +} + +impl OverlayFS { + pub fn ovl_upper_mnt(&self) -> Arc { + self.layers[0].mnt.clone() + } + pub fn make_overlayfs( + data: Option<&dyn FileSystemMakerData>, + ) -> Result, SystemError> { + let mount_data = data + .and_then(|d| d.as_any().downcast_ref::()) + .ok_or(SystemError::EINVAL)?; + + let upper_inode = ROOT_INODE() + .lookup(&mount_data.upper_dir) + .map_err(|_| SystemError::EINVAL)?; + let upper_layer = OvlLayer { + mnt: Arc::new(OvlInode::new( + mount_data.upper_dir.clone(), + Some(upper_inode), + None, + )), + index: 0, + fsid: 0, + }; + + let lower_layers: Result, SystemError> = mount_data + .lower_dirs + .iter() + .enumerate() + .map(|(i, dir)| { + let lower_inode = ROOT_INODE().lookup(dir).map_err(|_| SystemError::EINVAL)?; // 处理错误 + Ok(OvlLayer { + mnt: Arc::new(OvlInode::new(dir.clone(), None, Some(lower_inode))), + index: (i + 1) as u32, + fsid: (i + 1) as u32, + }) + }) + .collect(); + + let lower_layers = lower_layers?; + + let workdir = Arc::new(OvlInode::new(mount_data.work_dir.clone(), None, None)); + + if lower_layers.is_empty() { + return Err(SystemError::EINVAL); + } + + let mut layers = Vec::new(); + layers.push(upper_layer); + layers.extend(lower_layers); + + let root_inode = layers[0].mnt.clone(); + + let fs = OverlayFS { + numlayer: layers.len(), + numfs: 1, + numdatalayer: layers.len() - 1, + layers, + workdir, + root_inode, + }; + Ok(Arc::new(fs)) + } +} + +impl OvlInode { + pub fn ovl_lower_redirect(&self) -> Option<&str> { + if self.file_type == FileType::File || self.file_type == FileType::Dir { + Some(&self.redirect) + } else { + None + } + } + + pub fn create_whiteout(&self, name: &str) -> Result<(), SystemError> { + let whiteout_mode = vfs::syscall::ModeType::S_IFCHR; + let mut upper_inode = self.upper_inode.lock(); + if let Some(ref upper_inode) = *upper_inode { + upper_inode.mknod(name, whiteout_mode, WHITEOUT_DEV)?; + } else { + let new_inode = self + .fs + .upgrade() + .ok_or(SystemError::EROFS)? + .root_inode() + .create(name, FileType::CharDevice, whiteout_mode)?; + *upper_inode = Some(new_inode); + } + let mut flags = self.flags.lock(); + *flags |= WHITEOUT_FLAG; // 标记为 whiteout + Ok(()) + } + + fn is_whiteout(&self) -> bool { + let flags = self.flags.lock(); + self.file_type == FileType::CharDevice && (*flags & WHITEOUT_FLAG) != 0 + } + + fn has_whiteout(&self, name: &str) -> bool { + let upper_inode = self.upper_inode.lock(); + if let Some(ref upper_inode) = *upper_inode { + if let Ok(inode) = upper_inode.find(name) { + if let Some(ovl_inode) = inode.as_any_ref().downcast_ref::() { + return ovl_inode.is_whiteout(); + } + } + } + false + } +} + +impl IndexNode for OvlInode { + fn read_at( + &self, + offset: usize, + len: usize, + buf: &mut [u8], + data: crate::libs::spinlock::SpinLockGuard, + ) -> Result { + if let Some(ref upper_inode) = *self.upper_inode.lock() { + return upper_inode.read_at(offset, len, buf, data); + } + + if let Some(lower_inode) = &self.lower_inode { + return lower_inode.read_at(offset, len, buf, data); + } + + Err(SystemError::ENOENT) + } + + fn write_at( + &self, + offset: usize, + len: usize, + buf: &[u8], + data: crate::libs::spinlock::SpinLockGuard, + ) -> Result { + if (*self.upper_inode.lock()).is_none() { + self.copy_up()?; + } + if let Some(ref upper_inode) = *self.upper_inode.lock() { + return upper_inode.write_at(offset, len, buf, data); + } + + Err(SystemError::EROFS) + } + + fn fs(&self) -> Arc { + self.fs.upgrade().unwrap() + } + + fn metadata(&self) -> Result { + if let Some(ref upper_inode) = *self.upper_inode.lock() { + return upper_inode.metadata(); + } + + if let Some(ref lower_inode) = self.lower_inode { + return lower_inode.metadata(); + } + Ok(Metadata::default()) + } + + fn as_any_ref(&self) -> &dyn core::any::Any { + self + } + + fn list(&self) -> Result, system_error::SystemError> { + let mut entries: Vec = Vec::new(); + let upper_inode = self.upper_inode.lock(); + if let Some(ref upper_inode) = *upper_inode { + let upper_entries = upper_inode.list()?; + entries.extend(upper_entries); + } + if let Some(lower_inode) = &self.lower_inode { + let lower_entries = lower_inode.list()?; + for entry in lower_entries { + if !entries.contains(&entry) && !self.has_whiteout(&entry) { + entries.push(entry); + } + } + } + + Ok(entries) + } + + fn mkdir( + &self, + name: &str, + mode: vfs::syscall::ModeType, + ) -> Result, system_error::SystemError> { + if let Some(ref upper_inode) = *self.upper_inode.lock() { + upper_inode.mkdir(name, mode) + } else { + Err(SystemError::EROFS) + } + } + + fn rmdir(&self, name: &str) -> Result<(), SystemError> { + let upper_inode = self.upper_inode.lock(); + if let Some(ref upper_inode) = *upper_inode { + upper_inode.rmdir(name)?; + } else if let Some(lower_inode) = &self.lower_inode { + if lower_inode.find(name).is_ok() { + self.create_whiteout(name)?; + } else { + return Err(SystemError::ENOENT); + } + } else { + return Err(SystemError::ENOENT); + } + + Ok(()) + } + + fn unlink(&self, name: &str) -> Result<(), SystemError> { + let upper_inode = self.upper_inode.lock(); + if let Some(ref upper_inode) = *upper_inode { + upper_inode.unlink(name)?; + } else if let Some(lower_inode) = &self.lower_inode { + if lower_inode.find(name).is_ok() { + self.create_whiteout(name)?; + } else { + return Err(SystemError::ENOENT); + } + } else { + return Err(SystemError::ENOENT); + } + + Ok(()) + } + + fn link( + &self, + name: &str, + other: &Arc, + ) -> Result<(), system_error::SystemError> { + if let Some(ref upper_inode) = *self.upper_inode.lock() { + upper_inode.link(name, other) + } else { + Err(SystemError::EROFS) + } + } + + fn create( + &self, + name: &str, + file_type: vfs::FileType, + mode: vfs::syscall::ModeType, + ) -> Result, system_error::SystemError> { + if let Some(ref upper_inode) = *self.upper_inode.lock() { + upper_inode.create(name, file_type, mode) + } else { + Err(SystemError::EROFS) + } + } + + fn find(&self, name: &str) -> Result, system_error::SystemError> { + let upper_inode = self.upper_inode.lock(); + if let Some(ref upper) = *upper_inode { + if let Ok(inode) = upper.find(name) { + return Ok(inode); + } + } + if self.has_whiteout(name) { + return Err(SystemError::ENOENT); + } + + if let Some(lower) = &self.lower_inode { + if let Ok(inode) = lower.find(name) { + return Ok(inode); + } + } + + Err(SystemError::ENOENT) + } + + fn mknod( + &self, + filename: &str, + mode: vfs::syscall::ModeType, + dev_t: crate::driver::base::device::device_number::DeviceNumber, + ) -> Result, system_error::SystemError> { + let upper_inode = self.upper_inode.lock(); + if let Some(ref inode) = *upper_inode { + inode.mknod(filename, mode, dev_t) + } else { + Err(SystemError::EROFS) + } + } +} diff --git a/kernel/src/filesystem/ramfs/mod.rs b/kernel/src/filesystem/ramfs/mod.rs index 5f1ec4864..61dddf34f 100644 --- a/kernel/src/filesystem/ramfs/mod.rs +++ b/kernel/src/filesystem/ramfs/mod.rs @@ -1,7 +1,7 @@ use core::any::Any; use core::intrinsics::unlikely; -use crate::filesystem::vfs::FSMAKER; +use crate::filesystem::vfs::{FileSystemMakerData, FSMAKER}; use crate::libs::rwlock::RwLock; use crate::{ driver::base::device::device_number::DeviceNumber, @@ -35,7 +35,7 @@ const RAMFS_MAX_NAMELEN: usize = 64; const RAMFS_BLOCK_SIZE: u64 = 512; /// @brief 内存文件系统的Inode结构体 #[derive(Debug)] -struct LockedRamFSInode(SpinLock); +pub struct LockedRamFSInode(pub SpinLock); /// @brief 内存文件系统结构体 #[derive(Debug)] @@ -70,6 +70,35 @@ pub struct RamFSInode { name: DName, } +impl RamFSInode { + pub fn new() -> Self { + Self { + parent: Weak::default(), + self_ref: Weak::default(), + children: BTreeMap::new(), + data: Vec::new(), + metadata: Metadata { + dev_id: 0, + inode_id: generate_inode_id(), + size: 0, + blk_size: 0, + blocks: 0, + atime: PosixTimeSpec::default(), + mtime: PosixTimeSpec::default(), + ctime: PosixTimeSpec::default(), + file_type: FileType::Dir, + mode: ModeType::from_bits_truncate(0o777), + nlinks: 1, + uid: 0, + gid: 0, + raw_dev: DeviceNumber::default(), + }, + fs: Weak::default(), + special_node: None, + name: Default::default(), + } + } +} impl FileSystem for RamFS { fn root_inode(&self) -> Arc { return self.root_inode.clone(); @@ -105,31 +134,8 @@ impl RamFS { RAMFS_MAX_NAMELEN as u64, ); // 初始化root inode - let root: Arc = Arc::new(LockedRamFSInode(SpinLock::new(RamFSInode { - parent: Weak::default(), - self_ref: Weak::default(), - children: BTreeMap::new(), - data: Vec::new(), - metadata: Metadata { - dev_id: 0, - inode_id: generate_inode_id(), - size: 0, - blk_size: 0, - blocks: 0, - atime: PosixTimeSpec::default(), - mtime: PosixTimeSpec::default(), - ctime: PosixTimeSpec::default(), - file_type: FileType::Dir, - mode: ModeType::from_bits_truncate(0o777), - nlinks: 1, - uid: 0, - gid: 0, - raw_dev: DeviceNumber::default(), - }, - fs: Weak::default(), - special_node: None, - name: Default::default(), - }))); + let root: Arc = + Arc::new(LockedRamFSInode(SpinLock::new(RamFSInode::new()))); let result: Arc = Arc::new(RamFS { root_inode: root, @@ -147,7 +153,9 @@ impl RamFS { return result; } - pub fn make_ramfs() -> Result, SystemError> { + pub fn make_ramfs( + _data: Option<&dyn FileSystemMakerData>, + ) -> Result, SystemError> { let fs = RamFS::new(); return Ok(fs); } @@ -155,7 +163,10 @@ impl RamFS { #[distributed_slice(FSMAKER)] static RAMFSMAKER: FileSystemMaker = FileSystemMaker::new( "ramfs", - &(RamFS::make_ramfs as fn() -> Result, SystemError>), + &(RamFS::make_ramfs + as fn( + Option<&dyn FileSystemMakerData>, + ) -> Result, SystemError>), ); impl IndexNode for LockedRamFSInode { diff --git a/kernel/src/filesystem/vfs/mod.rs b/kernel/src/filesystem/vfs/mod.rs index 7801258e2..67b7643a3 100644 --- a/kernel/src/filesystem/vfs/mod.rs +++ b/kernel/src/filesystem/vfs/mod.rs @@ -935,12 +935,20 @@ impl FileSystemMaker { FileSystemMaker { function, name } } - pub fn call(&self) -> Result, SystemError> { - (self.function)() + pub fn call( + &self, + data: Option<&dyn FileSystemMakerData>, + ) -> Result, SystemError> { + (self.function)(data) } } -pub type FileSystemNewFunction = fn() -> Result, SystemError>; +pub trait FileSystemMakerData: Send + Sync { + fn as_any(&self) -> &dyn Any; +} + +pub type FileSystemNewFunction = + fn(data: Option<&dyn FileSystemMakerData>) -> Result, SystemError>; #[macro_export] macro_rules! define_filesystem_maker_slice { @@ -956,9 +964,18 @@ macro_rules! define_filesystem_maker_slice { /// 调用指定数组中的所有初始化器 #[macro_export] macro_rules! producefs { - ($initializer_slice:ident,$filesystem:ident) => { + ($initializer_slice:ident,$filesystem:ident,$raw_data : ident) => { match $initializer_slice.iter().find(|&m| m.name == $filesystem) { - Some(maker) => maker.call(), + Some(maker) => { + let mount_data = match $filesystem { + "overlay" => OverlayMountData::from_row($raw_data).ok(), + _ => None, + }; + let data: Option<&dyn FileSystemMakerData> = + mount_data.as_ref().map(|d| d as &dyn FileSystemMakerData); + + maker.call(data) + } None => { log::error!("mismatch filesystem type : {}", $filesystem); Err(SystemError::EINVAL) diff --git a/kernel/src/filesystem/vfs/syscall.rs b/kernel/src/filesystem/vfs/syscall.rs index ef8220ef7..795469bd4 100644 --- a/kernel/src/filesystem/vfs/syscall.rs +++ b/kernel/src/filesystem/vfs/syscall.rs @@ -1,4 +1,5 @@ -use core::ffi::c_void; +use crate::filesystem::overlayfs::OverlayMountData; +use crate::filesystem::vfs::FileSystemMakerData; use core::mem::size_of; use alloc::{string::String, sync::Arc, vec::Vec}; @@ -1706,7 +1707,7 @@ impl Syscall { target: *const u8, filesystemtype: *const u8, _mountflags: usize, - _data: *const c_void, + data: *const u8, ) -> Result { let target = user_access::check_and_clone_cstr(target, Some(MAX_PATHLEN))? .into_string() @@ -1715,7 +1716,7 @@ impl Syscall { let fstype_str = user_access::check_and_clone_cstr(filesystemtype, Some(MAX_PATHLEN))?; let fstype_str = fstype_str.to_str().map_err(|_| SystemError::EINVAL)?; - let fstype = producefs!(FSMAKER, fstype_str)?; + let fstype = producefs!(FSMAKER, fstype_str, data)?; Vcore::do_mount(fstype, &target)?; diff --git a/kernel/src/init/initial_kthread.rs b/kernel/src/init/initial_kthread.rs index 0d71c8d43..25fb21911 100644 --- a/kernel/src/init/initial_kthread.rs +++ b/kernel/src/init/initial_kthread.rs @@ -10,6 +10,7 @@ use crate::{ arch::{interrupt::TrapFrame, process::arch_switch_to_user}, driver::{net::e1000e::e1000e::e1000e_init, virtio::virtio::virtio_probe}, filesystem::vfs::core::mount_root_fs, + namespaces::NsProxy, net::net_core::net_init, process::{ exec::ProcInitInfo, kthread::KernelThreadMechanism, stdio::stdio_init, ProcessFlags, @@ -140,6 +141,7 @@ fn run_init_process( trap_frame: &mut TrapFrame, ) -> Result<(), SystemError> { compiler_fence(Ordering::SeqCst); + ProcessManager::current_pcb().set_nsproxy(NsProxy::new()); // 初始化init进程的namespace let path = proc_init_info.proc_name.to_str().unwrap(); Syscall::do_execve( @@ -148,5 +150,6 @@ fn run_init_process( proc_init_info.envs.clone(), trap_frame, )?; + Ok(()) } diff --git a/kernel/src/lib.rs b/kernel/src/lib.rs index 5a7e09535..881f6d319 100644 --- a/kernel/src/lib.rs +++ b/kernel/src/lib.rs @@ -48,6 +48,7 @@ mod libs; #[macro_use] mod include; mod bpf; +mod cgroup; mod debug; mod driver; // 如果driver依赖了libs，应该在libs后面导出 mod exception; @@ -56,6 +57,7 @@ mod init; mod ipc; mod misc; mod mm; +mod namespaces; mod net; mod perf; mod process; @@ -93,8 +95,6 @@ extern crate wait_queue_macros; use crate::mm::allocator::kernel_allocator::KernelAllocator; -use crate::process::ProcessManager; - #[cfg(all(feature = "backtrace", target_arch = "x86_64"))] extern crate mini_backtrace; @@ -112,6 +112,7 @@ pub static KERNEL_ALLOCATOR: KernelAllocator = KernelAllocator; #[no_mangle] pub fn panic(info: &PanicInfo) -> ! { use log::error; + use process::ProcessManager; error!("Kernel Panic Occurred."); diff --git a/kernel/src/libs/ida/src/lib.rs b/kernel/src/libs/ida/src/lib.rs new file mode 100644 index 000000000..ac25d557c --- /dev/null +++ b/kernel/src/libs/ida/src/lib.rs @@ -0,0 +1,55 @@ +#![no_std] +#![feature(core_intrinsics)] +#![allow(clippy::needless_return)] + +use core::intrinsics::unlikely; +use core::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; + +/// id分配器 +/// +/// TODO: 当前只是为了简单实现功能，将来这里应使用类似linux的ida的方式去实现 +#[derive(Debug)] +pub struct IdAllocator { + current_id: AtomicUsize, + max_id: usize, + dead: AtomicBool, +} + +impl IdAllocator { + /// 创建一个新的id分配器 + pub const fn new(initial_id: usize, max_id: usize) -> Self { + Self { + current_id: AtomicUsize::new(initial_id), + max_id, + dead: AtomicBool::new(false), + } + } + + /// 分配一个新的id + /// + /// ## 返回 + /// + /// 如果分配成功，返回Some(id)，否则返回None + pub fn alloc(&self) -> Option { + if unlikely(self.dead.load(Ordering::SeqCst)) { + return None; + } + + let ret = self.current_id.fetch_add(1, Ordering::SeqCst); + // 如果id溢出，panic + if ret == self.max_id { + self.dead.store(true, Ordering::SeqCst); + return None; + } + + return Some(ret); + } + + pub fn free(&self, _id: usize) { + // todo: free + } + + pub fn get_max_id(&self) -> usize { + self.max_id + } +} diff --git a/kernel/src/namespaces/mnt_namespace.rs b/kernel/src/namespaces/mnt_namespace.rs new file mode 100644 index 000000000..e5312d82c --- /dev/null +++ b/kernel/src/namespaces/mnt_namespace.rs @@ -0,0 +1,206 @@ +#![allow(dead_code, unused_variables, unused_imports)] +use core::sync::atomic::AtomicU64; +use core::sync::atomic::Ordering; + +use alloc::boxed::Box; +use alloc::string::ToString; + +use alloc::string::String; + +use alloc::sync::Arc; +use system_error::SystemError; + +use super::namespace::Namespace; +use super::namespace::NsOperations; +use super::ucount::Ucount::MntNamespaces; +use super::{namespace::NsCommon, ucount::UCounts, user_namespace::UserNamespace}; +use crate::container_of; +use crate::filesystem::vfs::mount::MountFSInode; +use crate::filesystem::vfs::IndexNode; +use crate::filesystem::vfs::InodeId; +use crate::filesystem::vfs::MountFS; +use crate::filesystem::vfs::ROOT_INODE; +use crate::libs::rbtree::RBTree; +use crate::libs::wait_queue::WaitQueue; +use crate::process::fork::CloneFlags; +use crate::process::ProcessManager; +use crate::syscall::Syscall; +#[allow(dead_code)] +#[derive(Debug)] +pub struct MntNamespace { + /// namespace 共有的部分 + ns_common: Arc, + /// 关联的用户名字空间 + user_ns: Arc, + /// 资源计数器 + ucounts: Arc, + /// 根文件系统 + root: Option>, + /// 红黑树用于挂载所有挂载点 + mounts: RBTree, + /// 等待队列 + poll: WaitQueue, + /// 挂载序列号 + seq: AtomicU64, + /// 挂载点的数量 + nr_mounts: u32, + /// 待处理的挂载点 + pending_mounts: u32, +} + +impl Default for MntNamespace { + fn default() -> Self { + Self::new() + } +} + +#[derive(Debug)] +struct MntNsOperations { + name: String, + clone_flags: CloneFlags, +} + +/// 使用该结构体的时候加spinlock +#[derive(Clone, Debug)] +pub struct FsStruct { + umask: u32, //文件权限掩码 + pub root: Arc, + pub pwd: Arc, +} +impl Default for FsStruct { + fn default() -> Self { + Self::new() + } +} + +impl FsStruct { + pub fn new() -> Self { + Self { + umask: 0o22, + root: ROOT_INODE(), + pwd: ROOT_INODE(), + } + } + pub fn set_root(&mut self, inode: Arc) { + self.root = inode; + } + pub fn set_pwd(&mut self, inode: Arc) { + self.pwd = inode; + } +} + +impl Namespace for MntNamespace { + fn ns_common_to_ns(ns_common: Arc) -> Arc { + let ns_common_ptr = Arc::as_ptr(&ns_common); + container_of!(ns_common_ptr, MntNamespace, ns_common) + } +} + +impl MntNsOperations { + pub fn new(name: String) -> Self { + Self { + name, + clone_flags: CloneFlags::CLONE_NEWNS, + } + } +} + +impl NsOperations for MntNsOperations { + fn get(&self, pid: crate::process::Pid) -> Option> { + let pcb = ProcessManager::find(pid); + pcb.map(|pcb| pcb.get_nsproxy().read().mnt_namespace.ns_common.clone()) + } + // 不存在这个方法 + fn get_parent(&self, _ns_common: Arc) -> Result, SystemError> { + unreachable!() + } + fn install( + &self, + nsset: &mut super::NsSet, + ns_common: Arc, + ) -> Result<(), SystemError> { + let nsproxy = &mut nsset.nsproxy; + let mnt_ns = MntNamespace::ns_common_to_ns(ns_common); + if mnt_ns.is_anon_ns() { + return Err(SystemError::EINVAL); + } + nsproxy.mnt_namespace = mnt_ns; + + nsset.fs.lock().set_pwd(ROOT_INODE()); + nsset.fs.lock().set_root(ROOT_INODE()); + Ok(()) + } + fn owner(&self, ns_common: Arc) -> Arc { + let mnt_ns = MntNamespace::ns_common_to_ns(ns_common); + mnt_ns.user_ns.clone() + } + fn put(&self, ns_common: Arc) { + let pid_ns = MntNamespace::ns_common_to_ns(ns_common); + } +} +impl MntNamespace { + pub fn new() -> Self { + let ns_common = Arc::new(NsCommon::new(Box::new(MntNsOperations::new( + "mnt".to_string(), + )))); + + Self { + ns_common, + user_ns: Arc::new(UserNamespace::new()), + ucounts: Arc::new(UCounts::new()), + root: None, + mounts: RBTree::new(), + poll: WaitQueue::default(), + seq: AtomicU64::new(0), + nr_mounts: 0, + pending_mounts: 0, + } + } + /// anon 用来判断是否是匿名的.匿名函数的问题还需要考虑 + pub fn create_mnt_namespace( + &self, + user_ns: Arc, + anon: bool, + ) -> Result { + let ucounts = self.inc_mnt_namespace(user_ns.clone())?; + if ucounts.is_none() { + return Err(SystemError::ENOSPC); + } + let ucounts = ucounts.unwrap(); + let ns_common = Arc::new(NsCommon::new(Box::new(MntNsOperations::new( + "mnt".to_string(), + )))); + let seq = AtomicU64::new(0); + if !anon { + seq.fetch_add(1, core::sync::atomic::Ordering::SeqCst); + } + Ok(Self { + ns_common, + user_ns, + ucounts, + root: None, + mounts: RBTree::new(), + poll: WaitQueue::default(), + seq, + nr_mounts: 0, + pending_mounts: 0, + }) + } + + pub fn inc_mnt_namespace( + &self, + user_ns: Arc, + ) -> Result>, SystemError> { + Ok(self + .ucounts + .inc_ucounts(user_ns, Syscall::geteuid()?, MntNamespaces)) + } + + pub fn dec_mnt_namespace(&self, uc: Arc) { + UCounts::dec_ucount(uc, super::ucount::Ucount::MntNamespaces) + } + //判断是不是匿名空间 + pub fn is_anon_ns(&self) -> bool { + self.seq.load(Ordering::SeqCst) == 0 + } +} diff --git a/kernel/src/namespaces/mod.rs b/kernel/src/namespaces/mod.rs new file mode 100644 index 000000000..a9043a125 --- /dev/null +++ b/kernel/src/namespaces/mod.rs @@ -0,0 +1,92 @@ +use alloc::sync::Arc; +use mnt_namespace::{FsStruct, MntNamespace}; +use pid_namespace::PidNamespace; +use system_error::SystemError; +use user_namespace::UserNamespace; + +use crate::{ + libs::spinlock::SpinLock, + process::{fork::CloneFlags, ProcessControlBlock}, +}; + +pub mod mnt_namespace; +pub mod namespace; +pub mod pid_namespace; +pub mod syscall; +pub mod ucount; +pub mod user_namespace; + +/// 管理 namespace,包含了所有namespace的信息 +#[derive(Clone)] +pub struct NsSet { + flags: u64, + nsproxy: NsProxy, + pub fs: Arc>, +} +#[derive(Debug, Clone)] +pub struct NsProxy { + pub pid_namespace: Arc, + pub mnt_namespace: Arc, +} +impl Default for NsProxy { + fn default() -> Self { + Self::new() + } +} + +impl NsProxy { + pub fn new() -> Self { + Self { + pid_namespace: Arc::new(PidNamespace::new()), + mnt_namespace: Arc::new(MntNamespace::new()), + } + } + pub fn set_pid_namespace(&mut self, new_pid_ns: Arc) { + self.pid_namespace = new_pid_ns; + } + + pub fn set_mnt_namespace(&mut self, new_mnt_ns: Arc) { + self.mnt_namespace = new_mnt_ns; + } +} + +pub fn create_new_namespaces( + clone_flags: u64, + pcb: &Arc, + user_ns: Arc, +) -> Result { + let mut nsproxy = NsProxy::new(); + // pid_namespace + let new_pid_ns = if (clone_flags & CloneFlags::CLONE_NEWPID.bits()) != 0 { + Arc::new(PidNamespace::new().create_pid_namespace( + pcb.get_nsproxy().read().pid_namespace.clone(), + user_ns.clone(), + )?) + } else { + pcb.get_nsproxy().read().pid_namespace.clone() + }; + nsproxy.set_pid_namespace(new_pid_ns); + + // mnt_namespace + let new_mnt_ns = if clone_flags & CloneFlags::CLONE_NEWNS.bits() != 0 { + Arc::new(MntNamespace::new().create_mnt_namespace(user_ns.clone(), false)?) + } else { + pcb.get_nsproxy().read().mnt_namespace.clone() + }; + nsproxy.set_mnt_namespace(new_mnt_ns); + + Ok(nsproxy) +} + +#[macro_export] +macro_rules! container_of { + ($ptr:expr, $struct:path, $field:ident) => { + unsafe { + let dummy = core::mem::MaybeUninit::<$struct>::uninit(); + let dummy_ptr = dummy.as_ptr(); + let field_ptr = &(*dummy_ptr).$field as *const _ as usize; + let offset = field_ptr - dummy_ptr as usize; + Arc::from_raw(($ptr as *const u8).wrapping_sub(offset) as *mut $struct) + } + }; +} diff --git a/kernel/src/namespaces/namespace.rs b/kernel/src/namespaces/namespace.rs new file mode 100644 index 000000000..06b0e6aca --- /dev/null +++ b/kernel/src/namespaces/namespace.rs @@ -0,0 +1,119 @@ +#![allow(dead_code, unused_variables, unused_imports)] +use core::fmt::Debug; + +use crate::filesystem::procfs::ProcFSInode; +use crate::filesystem::vfs::{IndexNode, ROOT_INODE}; +use crate::namespaces::user_namespace::UserNamespace; +use crate::process::fork::CloneFlags; +use crate::process::{Pid, ProcessControlBlock, ProcessManager}; +use alloc::boxed::Box; +use alloc::sync::Arc; +use system_error::SystemError; + +// 目前无credit功能，采用全局静态的user_namespace +lazy_static! { + pub static ref USER_NS: Arc = Arc::new(UserNamespace::new()); +} +use super::{create_new_namespaces, NsProxy, NsSet}; +pub trait NsOperations: Send + Sync + Debug { + fn get(&self, pid: Pid) -> Option>; + fn put(&self, ns_common: Arc); + fn install(&self, nsset: &mut NsSet, ns_common: Arc) -> Result<(), SystemError>; + fn owner(&self, ns_common: Arc) -> Arc; + fn get_parent(&self, ns_common: Arc) -> Result, SystemError>; +} +#[derive(Debug)] +pub struct NsCommon { + ops: Box, + stashed: Arc, +} + +impl NsCommon { + pub fn new(ops: Box) -> Self { + let inode = ROOT_INODE().find("proc").unwrap_or_else(|_| ROOT_INODE()); + Self { + ops, + stashed: inode, + } + } +} + +pub enum NsType { + Pid, + User, + Uts, + Ipc, + Net, + Mnt, + Cgroup, + Time, +} + +pub trait Namespace { + fn ns_common_to_ns(ns_common: Arc) -> Arc; +} + +pub fn check_unshare_flags(unshare_flags: u64) -> Result { + let valid_flags = CloneFlags::CLONE_THREAD + | CloneFlags::CLONE_FS + | CloneFlags::CLONE_NEWNS + | CloneFlags::CLONE_SIGHAND + | CloneFlags::CLONE_VM + | CloneFlags::CLONE_FILES + | CloneFlags::CLONE_SYSVSEM + | CloneFlags::CLONE_NEWUTS + | CloneFlags::CLONE_NEWIPC + | CloneFlags::CLONE_NEWNET + | CloneFlags::CLONE_NEWUSER + | CloneFlags::CLONE_NEWPID + | CloneFlags::CLONE_NEWCGROUP; + + if unshare_flags & !valid_flags.bits() != 0 { + return Err(SystemError::EINVAL); + } + Ok(0) +} + +pub fn unshare_nsproxy_namespaces(unshare_flags: u64) -> Result, SystemError> { + if (unshare_flags + & (CloneFlags::CLONE_NEWNS.bits() + | CloneFlags::CLONE_NEWUTS.bits() + | CloneFlags::CLONE_NEWIPC.bits() + | CloneFlags::CLONE_NEWNET.bits() + | CloneFlags::CLONE_NEWPID.bits() + | CloneFlags::CLONE_NEWCGROUP.bits())) + == 0 + { + return Ok(None); + } + let current = ProcessManager::current_pid(); + let pcb = ProcessManager::find(current).unwrap(); + let new_nsproxy = create_new_namespaces(unshare_flags, &pcb, USER_NS.clone())?; + Ok(Some(new_nsproxy)) +} + +pub fn switch_task_namespace(pcb: Arc, new_nsproxy: NsProxy) { + let ns = pcb.get_nsproxy(); + pcb.set_nsproxy(new_nsproxy); +} + +pub fn prepare_nsset(flags: u64) -> Result { + let current = ProcessManager::current_pcb(); + Ok(NsSet { + flags, + fs: current.fs_struct(), + nsproxy: create_new_namespaces(flags, ¤t, USER_NS.clone())?, + }) +} + +pub fn commit_nsset(nsset: NsSet) { + let flags = CloneFlags::from_bits_truncate(nsset.flags); + let current = ProcessManager::current_pcb(); + if flags.contains(CloneFlags::CLONE_NEWNS) { + let fs = current.fs_struct(); + let nsset_fs = nsset.fs.lock(); + fs.lock().set_pwd(nsset_fs.pwd.clone()); + fs.lock().set_root(nsset_fs.root.clone()); + } + switch_task_namespace(current, nsset.nsproxy); // 转移所有权 +} diff --git a/kernel/src/namespaces/pid_namespace.rs b/kernel/src/namespaces/pid_namespace.rs new file mode 100644 index 000000000..7f1eb05b2 --- /dev/null +++ b/kernel/src/namespaces/pid_namespace.rs @@ -0,0 +1,273 @@ +#![allow(dead_code, unused_variables, unused_imports)] +use alloc::vec::Vec; + +use super::namespace::Namespace; +use super::ucount::Ucount::PidNamespaces; +use super::NsSet; +use super::{namespace::NsCommon, ucount::UCounts, user_namespace::UserNamespace}; +use crate::container_of; +use crate::filesystem::vfs::{IndexNode, ROOT_INODE}; +use crate::namespaces::namespace::NsOperations; +use crate::process::fork::CloneFlags; +use crate::process::ProcessManager; +use crate::syscall::Syscall; +use crate::{libs::rwlock::RwLock, process::Pid}; +use alloc::boxed::Box; +use alloc::string::String; +use alloc::string::ToString; +use alloc::sync::Arc; +use ida::IdAllocator; +use system_error::SystemError; +use system_error::SystemError::ENOSPC; + +const INT16_MAX: u32 = 32767; +const MAX_PID_NS_LEVEL: usize = 32; +const PIDNS_ADDING: u32 = 1 << 31; +const PID_MAX: usize = 4096; +static PID_IDA: ida::IdAllocator = ida::IdAllocator::new(1, usize::MAX).unwrap(); +#[derive(Debug)] +#[repr(C)] +pub struct PidNamespace { + id_alloctor: RwLock, + /// 已经分配的进程数 + pid_allocated: u32, + /// 当前的pid_namespace所在的层数 + pub level: usize, + /// 父命名空间 + parent: Option>, + /// 资源计数器 + ucounts: Arc, + /// 关联的用户namespace + user_ns: Arc, + /// 回收孤儿进程的init进程 + child_reaper: Arc>, + /// namespace共有部分 + pub ns_common: Arc, +} + +impl Default for PidNamespace { + fn default() -> Self { + Self::new() + } +} + +#[derive(Debug, Clone)] +pub struct PidStrcut { + pub level: usize, + pub numbers: Vec, + pub stashed: Arc, +} + +impl Default for PidStrcut { + fn default() -> Self { + Self::new() + } +} +#[derive(Debug, Clone)] +pub struct UPid { + pub nr: Pid, // 在该pid_namespace 中的pid + pub ns: Arc, +} + +impl PidStrcut { + pub fn new() -> Self { + Self { + level: 0, + numbers: vec![UPid { + nr: Pid::new(0), + ns: Arc::new(PidNamespace::new()), + }], + stashed: ROOT_INODE(), + } + } + + pub fn put_pid(pid: PidStrcut) { + let ns = pid.numbers[pid.level].ns.clone(); + let id = pid.numbers[pid.level].nr.data(); + ns.id_alloctor.write().free(id); + } + pub fn alloc_pid(ns: Arc, set_tid: Vec) -> Result { + let mut set_tid_size = set_tid.len(); + if set_tid_size > ns.level + 1 { + return Err(SystemError::EINVAL); + } + + let mut numbers = Vec::::with_capacity(ns.level + 1); + let mut tid_iter = set_tid.into_iter().rev(); + let mut pid_ns = ns.clone(); // 当前正在处理的命名空间 + for i in (0..=ns.level).rev() { + let tid = tid_iter.next().unwrap_or(0); + if set_tid_size > 0 { + if tid < 1 || tid > INT16_MAX as usize { + return Err(SystemError::EINVAL); + } + set_tid_size -= 1; + } + let mut nr = tid; + + if tid == 0 { + nr = pid_ns + .id_alloctor + .write() + .alloc() + .expect("PID allocation failed."); + } + + numbers.insert( + i, + UPid { + nr: Pid::from(nr), + ns: pid_ns.clone(), + }, + ); + + if let Some(parent_ns) = &pid_ns.parent { + pid_ns = parent_ns.clone(); + } else { + break; // 根命名空间，无需继续向上。 + } + } + Ok(PidStrcut { + level: ns.level, + numbers, + stashed: ROOT_INODE(), + }) + } + + pub fn ns_of_pid(&self) -> Arc { + self.numbers[self.level].ns.clone() + } +} +#[derive(Debug)] +struct PidNsOperations { + name: String, + clone_flags: CloneFlags, +} +impl PidNsOperations { + pub fn new(name: String) -> Self { + Self { + name, + clone_flags: CloneFlags::CLONE_NEWPID, + } + } +} +impl Namespace for PidNamespace { + fn ns_common_to_ns(ns_common: Arc) -> Arc { + container_of!(Arc::as_ptr(&ns_common), PidNamespace, ns_common) + } +} + +impl NsOperations for PidNsOperations { + fn put(&self, ns_common: Arc) { + let _pid_ns = PidNamespace::ns_common_to_ns(ns_common); + // pid_ns 超出作用域自动drop 同时递归drop + } + + fn owner(&self, ns_common: Arc) -> Arc { + let pid_ns = PidNamespace::ns_common_to_ns(ns_common); + pid_ns.user_ns.clone() + } + + fn get_parent(&self, ns_common: Arc) -> Result, SystemError> { + let current = ProcessManager::current_pid(); + let pcb = ProcessManager::find(current).unwrap(); + let active = pcb.pid_strcut().read().ns_of_pid(); + let mut pid_ns = &PidNamespace::ns_common_to_ns(ns_common).parent; + + while let Some(ns) = pid_ns { + if Arc::ptr_eq(&active, ns) { + return Ok(ns.ns_common.clone()); + } + pid_ns = &ns.parent; + } + Err(SystemError::EPERM) + } + + fn get(&self, pid: Pid) -> Option> { + let pcb = ProcessManager::find(pid); + pcb.map(|pcb| pcb.get_nsproxy().read().pid_namespace.ns_common.clone()) + } + fn install(&self, nsset: &mut NsSet, ns_common: Arc) -> Result<(), SystemError> { + let nsproxy = &mut nsset.nsproxy; + let current = ProcessManager::current_pid(); + let pcb = ProcessManager::find(current).unwrap(); + let active = pcb.pid_strcut().read().ns_of_pid(); + let mut pid_ns = PidNamespace::ns_common_to_ns(ns_common); + if pid_ns.level < active.level { + return Err(SystemError::EINVAL); + } + while pid_ns.level > active.level { + if let Some(ns) = &pid_ns.parent { + pid_ns = ns.clone(); + } else { + break; + } + } + if Arc::ptr_eq(&pid_ns, &active) { + return Err(SystemError::EINVAL); + } + nsproxy.pid_namespace = pid_ns.clone(); + Ok(()) + } +} +impl PidNamespace { + pub fn new() -> Self { + Self { + id_alloctor: RwLock::new(IdAllocator::new(1, PID_MAX).unwrap()), + pid_allocated: 1, + level: 0, + child_reaper: Arc::new(RwLock::new(Pid::from(1))), + parent: None, + ucounts: Arc::new(UCounts::new()), + user_ns: Arc::new(UserNamespace::new()), + ns_common: Arc::new(NsCommon::new(Box::new(PidNsOperations::new( + "pid".to_string(), + )))), + } + } + + pub fn create_pid_namespace( + &self, + parent: Arc, + user_ns: Arc, + ) -> Result { + let level = parent.level + 1; + if level > MAX_PID_NS_LEVEL { + return Err(ENOSPC); + } + let ucounts = self.inc_pid_namespaces(user_ns.clone())?; + + if ucounts.is_none() { + return Err(SystemError::ENOSPC); + } + let ucounts = ucounts.unwrap(); + + let ns_common = Arc::new(NsCommon::new(Box::new(PidNsOperations::new( + "pid".to_string(), + )))); + let child_reaper = parent.child_reaper.clone(); + Ok(Self { + id_alloctor: RwLock::new(IdAllocator::new(1, PID_MAX).unwrap()), + pid_allocated: PIDNS_ADDING, + level, + ucounts, + parent: Some(parent), + user_ns, + ns_common, + child_reaper, + }) + } + + pub fn inc_pid_namespaces( + &self, + user_ns: Arc, + ) -> Result>, SystemError> { + Ok(self + .ucounts + .inc_ucounts(user_ns, Syscall::geteuid()?, PidNamespaces)) + } + + pub fn dec_pid_namespaces(&mut self, uc: Arc) { + UCounts::dec_ucount(uc, PidNamespaces) + } +} diff --git a/kernel/src/namespaces/syscall.rs b/kernel/src/namespaces/syscall.rs new file mode 100644 index 000000000..fe6143dc7 --- /dev/null +++ b/kernel/src/namespaces/syscall.rs @@ -0,0 +1,50 @@ +use system_error::SystemError; + +use crate::{ + process::{fork::CloneFlags, ProcessManager}, + syscall::Syscall, +}; + +use super::namespace::{ + check_unshare_flags, commit_nsset, prepare_nsset, unshare_nsproxy_namespaces, +}; + +impl Syscall { + pub fn sys_unshare(mut unshare_flags: u64) -> Result { + if unshare_flags & CloneFlags::CLONE_NEWUSER.bits() != 0 { + unshare_flags |= CloneFlags::CLONE_THREAD.bits() | CloneFlags::CLONE_FS.bits(); + } + + if unshare_flags & CloneFlags::CLONE_VM.bits() != 0 { + unshare_flags |= CloneFlags::CLONE_SIGHAND.bits(); + } + + if unshare_flags & CloneFlags::CLONE_SIGHAND.bits() != 0 { + unshare_flags |= CloneFlags::CLONE_THREAD.bits(); + } + + if unshare_flags & CloneFlags::CLONE_NEWNS.bits() != 0 { + unshare_flags |= CloneFlags::CLONE_FS.bits(); + } + + let check = check_unshare_flags(unshare_flags)?; + + let current = ProcessManager::current_pcb(); + if let Some(nsproxy) = unshare_nsproxy_namespaces(unshare_flags)? { + *current.get_nsproxy().write() = nsproxy; + } + + Ok(check) + } + + pub fn sys_setns(_fd: i32, flags: u64) -> Result { + let check = check_unshare_flags(flags)?; + + let nsset = prepare_nsset(flags)?; + + if check == 0 { + commit_nsset(nsset) + }; + Ok(0) + } +} diff --git a/kernel/src/namespaces/ucount.rs b/kernel/src/namespaces/ucount.rs new file mode 100644 index 000000000..358c8cef0 --- /dev/null +++ b/kernel/src/namespaces/ucount.rs @@ -0,0 +1,187 @@ +#![allow(dead_code, unused_variables, unused_imports)] +use alloc::vec::Vec; +use core::{hash::Hash, sync::atomic::AtomicU32}; +use system_error::SystemError; + +use alloc::sync::Arc; +use hashbrown::HashMap; +use log::warn; + +use super::user_namespace::UserNamespace; +use crate::libs::mutex::Mutex; + +#[derive(Clone, Copy)] +pub enum Ucount { + UserNamespaces = 1, + PidNamespaces = 2, + UtsNamespaces = 3, + IpcNamespaces = 4, + NetNamespaces = 5, + MntNamespaces = 6, + CgroupNamespaces = 7, + TimeNamespaces = 8, + Counts = 9, +} + +pub enum UcountRlimit { + Nproc = 1, + Msgqueue = 2, + Sigpending = 3, + Memlock = 4, + Counts = 5, +} + +lazy_static! { + static ref COUNT_MANAGER: Arc = Arc::new(CountManager::new()); +} + +#[derive(Debug)] +pub struct UCounts { + /// 对应的user_namespace + ns: Arc, + /// 用户标识符 + uid: usize, + count: AtomicU32, + ucount: Vec, //[AtomicU32; UCOUNT_COUNTS as usize], + rlimit: Vec, //[AtomicU32; UCOUNT_RLIMIT_COUNTS as usize], +} + +impl Default for UCounts { + fn default() -> Self { + Self::new() + } +} +impl UCounts { + pub fn new() -> Self { + Self { + ns: Arc::new(UserNamespace::new()), + uid: 0, + count: AtomicU32::new(1), + ucount: (0..Ucount::Counts as usize) + .map(|_| AtomicU32::new(0)) + .collect(), + rlimit: (0..UcountRlimit::Counts as usize) + .map(|_| AtomicU32::new(0)) + .collect(), + } + } + + fn alloc_ucounts(&self, ns: Arc, uid: usize) -> Arc { + let mut counts = COUNT_MANAGER.counts.lock(); + let key = UKey { + user_ns: ns.clone(), + uid, + }; + let uc = if let Some(uc) = counts.get(&key) { + self.count + .fetch_add(1, core::sync::atomic::Ordering::SeqCst); + uc.clone() + } else { + Arc::new(Self { + ns, + uid, + count: AtomicU32::new(1), + ucount: (0..Ucount::Counts as usize) + .map(|_| AtomicU32::new(0)) + .collect(), + rlimit: (0..UcountRlimit::Counts as usize) + .map(|_| AtomicU32::new(0)) + .collect(), + }) + }; + counts.insert(key, uc.clone()); + uc + } + + pub fn inc_ucounts( + &self, + user_ns: Arc, + uid: usize, + ucount_type: Ucount, + ) -> Option> { + let uc_type = ucount_type as usize; + let uc = self.alloc_ucounts(user_ns, uid); + let mut uc_iter = Some(uc.clone()); + let mut ucounts_add = vec![]; + while let Some(iter) = uc_iter { + let num = iter.ucount[uc_type].fetch_add(1, core::sync::atomic::Ordering::SeqCst); + ucounts_add.push(iter.clone()); + // 分配失败回滚 + if num > iter.ns.ucount_max[uc_type] { + for add_iter in &ucounts_add { + add_iter.ucount[uc_type].fetch_sub(1, core::sync::atomic::Ordering::SeqCst); + } + return None; + } + uc_iter = iter.ns.ucounts.clone(); + } + return Some(uc); + } + + fn find_ucounts(user_ns: Arc, uid: usize) -> Option> { + let counts = COUNT_MANAGER.counts.lock(); + let key = UKey { user_ns, uid }; + counts.get(&key).cloned() + } + + fn get_ucounts(uc: Arc) { + let mut counts = COUNT_MANAGER.counts.lock(); + let ukey = UKey { + user_ns: uc.ns.clone(), + uid: uc.uid, + }; + counts.insert(ukey, uc); + } + + pub fn dec_ucount(uc: Arc, ucount_type: Ucount) { + let mut uc_iter = Some(uc.clone()); + let uc_type = ucount_type as usize; + while let Some(iter) = uc_iter { + let num = iter.ucount[uc_type].fetch_sub(1, core::sync::atomic::Ordering::SeqCst); + if num == 0 { + warn!("count has reached zero"); + } + uc_iter = iter.ns.ucounts.clone(); + } + Self::put_ucounts(uc); + } + + fn put_ucounts(uc: Arc) { + let mut counts = COUNT_MANAGER.counts.lock(); + let key = UKey { + user_ns: uc.ns.clone(), + uid: uc.uid, + }; + counts.remove(&key); + } +} +struct UKey { + user_ns: Arc, + uid: usize, +} + +impl Hash for UKey { + fn hash(&self, state: &mut H) { + let user_ns_ptr = Arc::as_ptr(&self.user_ns); + user_ns_ptr.hash(state); + self.uid.hash(state) + } +} +impl Eq for UKey {} +impl PartialEq for UKey { + fn eq(&self, other: &Self) -> bool { + Arc::ptr_eq(&self.user_ns, &other.user_ns) && self.uid == other.uid + } +} + +struct CountManager { + counts: Mutex>>, +} + +impl CountManager { + fn new() -> Self { + Self { + counts: Mutex::new(HashMap::new()), + } + } +} diff --git a/kernel/src/namespaces/user_namespace.rs b/kernel/src/namespaces/user_namespace.rs new file mode 100644 index 000000000..2314ccfcb --- /dev/null +++ b/kernel/src/namespaces/user_namespace.rs @@ -0,0 +1,135 @@ +#![allow(dead_code, unused_variables, unused_imports)] + +use alloc::boxed::Box; + +use crate::libs::rwlock::RwLock; +use alloc::string::String; +use alloc::string::ToString; + +use alloc::vec::Vec; +use system_error::SystemError; + +use crate::namespaces::namespace::NsCommon; +use crate::namespaces::ucount::UCounts; +use crate::process::fork::CloneFlags; +use crate::process::Pid; +use alloc::sync::Arc; + +use super::namespace::NsOperations; +use super::ucount::Ucount::Counts; + +const UID_GID_MAP_MAX_BASE_EXTENTS: usize = 5; +const UCOUNT_MAX: u32 = 62636; +/// 管理用户ID和组ID的映射 +#[allow(dead_code)] +#[derive(Clone, Debug)] +struct UidGidMap { + nr_extents: u32, + extent: Vec, +} + +///区间映射 +#[allow(dead_code)] +#[derive(Clone, Debug)] +struct UidGidExtent { + first: u32, + lower_first: u32, + count: u32, +} +#[derive(Debug)] +pub struct UserNamespace { + uid_map: UidGidMap, + gid_map: UidGidMap, + progid_map: UidGidMap, + ///项目ID映射 + parent: Option>, + level: u32, + owner: usize, + group: usize, + ns_common: Arc, + flags: u32, + pid: Arc>, + pub ucounts: Option>, + pub ucount_max: Vec, //vec![u32; UCOUNT_COUNTS as usize], + pub rlimit_max: Vec, // vec![u32; UCOUNT_RLIMIT_COUNTS as usize], +} + +impl Default for UserNamespace { + fn default() -> Self { + Self::new() + } +} +#[derive(Debug)] +struct UserNsOperations { + name: String, + clone_flags: CloneFlags, +} +impl UserNsOperations { + pub fn new(name: String) -> Self { + Self { + name, + clone_flags: CloneFlags::CLONE_NEWUSER, + } + } +} +impl NsOperations for UserNsOperations { + fn get(&self, pid: Pid) -> Option> { + unimplemented!() + } + fn get_parent(&self, ns_common: Arc) -> Result, SystemError> { + unimplemented!() + } + fn install( + &self, + nsset: &mut super::NsSet, + ns_common: Arc, + ) -> Result<(), SystemError> { + unimplemented!() + } + fn owner(&self, ns_common: Arc) -> Arc { + unimplemented!() + } + fn put(&self, ns_common: Arc) { + unimplemented!() + } +} +impl UidGidMap { + pub fn new() -> Self { + Self { + nr_extents: 1, + extent: vec![UidGidExtent::new(); UID_GID_MAP_MAX_BASE_EXTENTS], + } + } +} + +impl UidGidExtent { + pub fn new() -> Self { + Self { + first: 0, + lower_first: 0, + count: u32::MAX, + } + } +} + +impl UserNamespace { + pub fn new() -> Self { + Self { + uid_map: UidGidMap::new(), + gid_map: UidGidMap::new(), + progid_map: UidGidMap::new(), + owner: 0, + level: 0, + group: 0, + flags: 1, + parent: None, + ns_common: Arc::new(NsCommon::new(Box::new(UserNsOperations::new( + "User".to_string(), + )))), + pid: Arc::new(RwLock::new(Pid::new(1))), + ucount_max: vec![UCOUNT_MAX; Counts as usize], + ucounts: None, + rlimit_max: vec![65535, 10, 32000, 64 * 1024], + } + } +} diff --git a/kernel/src/process/fork.rs b/kernel/src/process/fork.rs index 547325c6c..bde3b0b8e 100644 --- a/kernel/src/process/fork.rs +++ b/kernel/src/process/fork.rs @@ -1,3 +1,4 @@ +use alloc::vec::Vec; use core::{intrinsics::unlikely, sync::atomic::Ordering}; use alloc::{string::ToString, sync::Arc}; @@ -10,6 +11,7 @@ use crate::{ ipc::signal::flush_signal_handlers, libs::rwlock::RwLock, mm::VirtAddr, + namespaces::{create_new_namespaces, namespace::USER_NS, pid_namespace::PidStrcut}, process::ProcessFlags, sched::{sched_cgroup_fork, sched_fork}, smp::core::smp_get_processor_id, @@ -20,6 +22,7 @@ use super::{ kthread::{KernelThreadPcbPrivate, WorkerPrivate}, KernelStack, Pid, ProcessControlBlock, ProcessManager, }; +const MAX_PID_NS_LEVEL: usize = 32; bitflags! { /// 进程克隆标志 @@ -84,8 +87,8 @@ bitflags! { /// 因为这两个系统调用的参数很多，所以有这样一个载体更灵活 /// /// 仅仅作为参数传递 -#[derive(Debug, Clone, Copy)] #[allow(dead_code)] +#[derive(Debug, Clone)] pub struct KernelCloneArgs { pub flags: CloneFlags, @@ -93,7 +96,7 @@ pub struct KernelCloneArgs { pub pidfd: VirtAddr, pub child_tid: VirtAddr, pub parent_tid: VirtAddr, - pub set_tid: VirtAddr, + pub set_tid: Vec, /// 进程退出时发送的信号 pub exit_signal: Signal, @@ -122,7 +125,7 @@ impl KernelCloneArgs { pidfd: null_addr, child_tid: null_addr, parent_tid: null_addr, - set_tid: null_addr, + set_tid: Vec::with_capacity(MAX_PID_NS_LEVEL), exit_signal: Signal::SIGCHLD, stack: 0, stack_size: 0, @@ -260,6 +263,34 @@ impl ProcessManager { return Ok(()); } + #[inline(never)] + fn copy_namespaces( + clone_flags: &CloneFlags, + current_pcb: &Arc, + new_pcb: &Arc, + ) -> Result<(), SystemError> { + if !clone_flags.contains(CloneFlags::CLONE_NEWNS) + && !clone_flags.contains(CloneFlags::CLONE_NEWUTS) + && !clone_flags.contains(CloneFlags::CLONE_NEWIPC) + && !clone_flags.contains(CloneFlags::CLONE_NEWPID) + && !clone_flags.contains(CloneFlags::CLONE_NEWNET) + && !clone_flags.contains(CloneFlags::CLONE_NEWCGROUP) + { + new_pcb.set_nsproxy(current_pcb.get_nsproxy().read().clone()); + return Ok(()); + } + + if clone_flags.contains(CloneFlags::CLONE_NEWIPC) + && clone_flags.contains(CloneFlags::CLONE_SYSVSEM) + { + return Err(SystemError::EINVAL); + } + + let new_nsproxy = create_new_namespaces(clone_flags.bits(), current_pcb, USER_NS.clone())?; + *new_pcb.nsproxy.write() = new_nsproxy; + Ok(()) + } + #[inline(never)] fn copy_files( clone_flags: &CloneFlags, @@ -422,6 +453,11 @@ impl ProcessManager { ) }); + Self::copy_namespaces(&clone_flags, current_pcb, pcb).unwrap_or_else(|e|{ + panic!("fork: Failed to copy namespace form current process, current pid: [{:?}], new pid: [{:?}]. Error: {:?}", + current_pcb.pid(), pcb.pid(), e) + }); + // 拷贝文件描述符表 Self::copy_files(&clone_flags, current_pcb, pcb).unwrap_or_else(|e| { panic!( @@ -439,13 +475,19 @@ impl ProcessManager { }); // 拷贝线程 - Self::copy_thread(current_pcb, pcb, clone_args,current_trapframe).unwrap_or_else(|e| { + Self::copy_thread(current_pcb, pcb, &clone_args, current_trapframe).unwrap_or_else(|e| { panic!( "fork: Failed to copy thread from current process, current pid: [{:?}], new pid: [{:?}]. Error: {:?}", current_pcb.pid(), pcb.pid(), e ) }); - + if current_pcb.pid() != Pid(0) { + let new_pid = PidStrcut::alloc_pid( + pcb.get_nsproxy().read().pid_namespace.clone(), // 获取命名空间 + clone_args.set_tid.clone(), + )?; + *pcb.thread_pid.write() = new_pid; + } // 设置线程组id、组长 if clone_flags.contains(CloneFlags::CLONE_THREAD) { pcb.thread.write_irqsave().group_leader = diff --git a/kernel/src/process/mod.rs b/kernel/src/process/mod.rs index d5841e93b..1c5ce4bb4 100644 --- a/kernel/src/process/mod.rs +++ b/kernel/src/process/mod.rs @@ -50,11 +50,11 @@ use crate::{ ucontext::AddressSpace, VirtAddr, }, + namespaces::{mnt_namespace::FsStruct, pid_namespace::PidStrcut, NsProxy}, net::socket::SocketInode, - sched::completion::Completion, sched::{ - cpu_rq, fair::FairSchedEntity, prio::MAX_PRIO, DequeueFlag, EnqueueFlag, OnRq, SchedMode, - WakeupFlags, __schedule, + completion::Completion, cpu_rq, fair::FairSchedEntity, prio::MAX_PRIO, DequeueFlag, + EnqueueFlag, OnRq, SchedMode, WakeupFlags, __schedule, }, smp::{ core::smp_get_processor_id, @@ -90,7 +90,6 @@ pub static mut PROCESS_SWITCH_RESULT: Option> = None; /// 一个只改变1次的全局变量，标志进程管理器是否已经初始化完成 static mut __PROCESS_MANAGEMENT_INIT_DONE: bool = false; -#[derive(Debug)] pub struct SwitchResult { pub prev_pcb: Option>, pub next_pcb: Option>, @@ -609,14 +608,14 @@ bitflags! { const RANDOMIZE = 1 << 8; } } - #[derive(Debug)] pub struct ProcessControlBlock { /// 当前进程的pid pid: Pid, /// 当前进程的线程组id（这个值在同一个线程组内永远不变） tgid: Pid, - + /// 有关Pid的相关的信息 + thread_pid: Arc>, basic: RwLock, /// 当前进程的自旋锁持有计数 preempt_count: AtomicUsize, @@ -654,12 +653,18 @@ pub struct ProcessControlBlock { /// 线程信息 thread: RwLock, + /// 进程文件系统的状态 + fs: Arc>, + ///闹钟定时器 alarm_timer: SpinLock>, /// 进程的robust lock列表 robust_list: RwLock>, + /// namespace的指针 + nsproxy: Arc>, + /// 进程作为主体的凭证集 cred: SpinLock, } @@ -722,10 +727,10 @@ impl ProcessControlBlock { let ppcb: Weak = ProcessManager::find(ppid) .map(|p| Arc::downgrade(&p)) .unwrap_or_default(); - let pcb = Self { pid, tgid: pid, + thread_pid: Arc::new(RwLock::new(PidStrcut::new())), basic: basic_info, preempt_count, flags, @@ -742,8 +747,10 @@ impl ProcessControlBlock { children: RwLock::new(Vec::new()), wait_queue: WaitQueue::default(), thread: RwLock::new(ThreadInfo::new()), + fs: Arc::new(SpinLock::new(FsStruct::new())), alarm_timer: SpinLock::new(None), robust_list: RwLock::new(None), + nsproxy: Arc::new(RwLock::new(NsProxy::new())), cred: SpinLock::new(cred), }; @@ -886,11 +893,21 @@ impl ProcessControlBlock { return self.pid; } + #[inline(always)] + pub fn pid_strcut(&self) -> Arc> { + self.thread_pid.clone() + } + #[inline(always)] pub fn tgid(&self) -> Pid { return self.tgid; } + #[inline(always)] + pub fn fs_struct(&self) -> Arc> { + self.fs.clone() + } + /// 获取文件描述符表的Arc指针 #[inline(always)] pub fn fd_table(&self) -> Arc> { @@ -1022,6 +1039,14 @@ impl ProcessControlBlock { pub fn alarm_timer_irqsave(&self) -> SpinLockGuard> { return self.alarm_timer.lock_irqsave(); } + + pub fn get_nsproxy(&self) -> Arc> { + self.nsproxy.clone() + } + + pub fn set_nsproxy(&self, nsprsy: NsProxy) { + *self.nsproxy.write() = nsprsy; + } } impl Drop for ProcessControlBlock { diff --git a/kernel/src/process/syscall.rs b/kernel/src/process/syscall.rs index 1a2fec460..9dd04a114 100644 --- a/kernel/src/process/syscall.rs +++ b/kernel/src/process/syscall.rs @@ -188,7 +188,13 @@ impl Syscall { /// @brief 获取当前进程的pid pub fn getpid() -> Result { let current_pcb = ProcessManager::current_pcb(); - return Ok(current_pcb.tgid()); + // if let Some(pid_ns) = ¤t_pcb.get_nsproxy().read().pid_namespace { + // // 获取该进程在命名空间中的 PID + // return Ok(current_pcb.pid_strcut().read().numbers[pid_ns.level].nr); + // // 返回命名空间中的 PID + // } + // 默认返回 tgid + Ok(current_pcb.tgid()) } /// @brief 获取指定进程的pgid diff --git a/kernel/src/syscall/mod.rs b/kernel/src/syscall/mod.rs index 74f90610a..c83b67297 100644 --- a/kernel/src/syscall/mod.rs +++ b/kernel/src/syscall/mod.rs @@ -1,6 +1,5 @@ use core::{ ffi::{c_int, c_void}, - ptr::null, sync::atomic::{AtomicBool, Ordering}, }; @@ -1090,7 +1089,9 @@ impl Syscall { let source = args[0] as *const u8; let target = args[1] as *const u8; let filesystemtype = args[2] as *const u8; - return Self::mount(source, target, filesystemtype, 0, null()); + let mountflags = args[3]; + let data = args[4] as *const u8; // 额外的mount参数，实现自己的mountdata来获取 + return Self::mount(source, target, filesystemtype, mountflags, data); } SYS_UMOUNT2 => { @@ -1183,6 +1184,7 @@ impl Syscall { let flags = args[1] as u32; Self::sys_eventfd(initval, flags) } + SYS_UNSHARE => Self::sys_unshare(args[0] as u64), SYS_BPF => { let cmd = args[0] as u32; let attr = args[1] as *mut u8; diff --git a/user/apps/test_namespace/Cargo.toml b/user/apps/test_namespace/Cargo.toml new file mode 100644 index 000000000..9a287f068 --- /dev/null +++ b/user/apps/test_namespace/Cargo.toml @@ -0,0 +1,7 @@ +[package] +name = "test-namespace" +version = "0.1.0" +edition = "2021" + +[dependencies] +nix = { version = "0.29.0", features = ["sched", "process"] } diff --git a/user/apps/test_namespace/Makefile b/user/apps/test_namespace/Makefile new file mode 100644 index 000000000..1b0274d20 --- /dev/null +++ b/user/apps/test_namespace/Makefile @@ -0,0 +1,56 @@ +TOOLCHAIN="+nightly-2023-08-15-x86_64-unknown-linux-gnu" +RUSTFLAGS+="" + +ifdef DADK_CURRENT_BUILD_DIR +# 如果是在dadk中编译，那么安装到dadk的安装目录中 + INSTALL_DIR = $(DADK_CURRENT_BUILD_DIR) +else +# 如果是在本地编译，那么安装到当前目录下的install目录中 + INSTALL_DIR = ./install +endif + +ifeq ($(ARCH), x86_64) + export RUST_TARGET=x86_64-unknown-linux-musl +else ifeq ($(ARCH), riscv64) + export RUST_TARGET=riscv64gc-unknown-linux-gnu +else +# 默认为x86_86，用于本地编译 + export RUST_TARGET=x86_64-unknown-linux-musl +endif + +run: + RUSTFLAGS=$(RUSTFLAGS) cargo $(TOOLCHAIN) run --target $(RUST_TARGET) + +build: + RUSTFLAGS=$(RUSTFLAGS) cargo $(TOOLCHAIN) build --target $(RUST_TARGET) + +clean: + RUSTFLAGS=$(RUSTFLAGS) cargo $(TOOLCHAIN) clean --target $(RUST_TARGET) + +test: + RUSTFLAGS=$(RUSTFLAGS) cargo $(TOOLCHAIN) test --target $(RUST_TARGET) + +doc: + RUSTFLAGS=$(RUSTFLAGS) cargo $(TOOLCHAIN) doc --target $(RUST_TARGET) + +fmt: + RUSTFLAGS=$(RUSTFLAGS) cargo $(TOOLCHAIN) fmt + +fmt-check: + RUSTFLAGS=$(RUSTFLAGS) cargo $(TOOLCHAIN) fmt --check + +run-release: + RUSTFLAGS=$(RUSTFLAGS) cargo $(TOOLCHAIN) run --target $(RUST_TARGET) --release + +build-release: + RUSTFLAGS=$(RUSTFLAGS) cargo $(TOOLCHAIN) build --target $(RUST_TARGET) --release + +clean-release: + RUSTFLAGS=$(RUSTFLAGS) cargo $(TOOLCHAIN) clean --target $(RUST_TARGET) --release + +test-release: + RUSTFLAGS=$(RUSTFLAGS) cargo $(TOOLCHAIN) test --target $(RUST_TARGET) --release + +.PHONY: install +install: + RUSTFLAGS=$(RUSTFLAGS) cargo $(TOOLCHAIN) install --target $(RUST_TARGET) --path . --no-track --root $(INSTALL_DIR) --force diff --git a/user/apps/test_namespace/makefile.toml b/user/apps/test_namespace/makefile.toml new file mode 100644 index 000000000..dc3f667e6 --- /dev/null +++ b/user/apps/test_namespace/makefile.toml @@ -0,0 +1,63 @@ +# Makefile.toml + +[env] +TOOLCHAIN = "+nightly-2023-08-15-x86_64-unknown-linux-gnu" + +ARCH = { default = "x86_64" } +RUST_TARGET = { default = { if = "eq(env.ARCH, 'riscv64')", value = "riscv64gc-unknown-linux-gnu", else = "x86_64-unknown-linux-musl" } } +INSTALL_DIR = { default = { if = "defined(env.DADK_CURRENT_BUILD_DIR)", value = "${DADK_CURRENT_BUILD_DIR}", else = "./install" } } + +[tasks.build] +description = "Build the project" +command = "cargo" +args = ["${TOOLCHAIN}", "build", "--target", "${RUST_TARGET}"] + +[tasks.run] +description = "Run the project" +command = "cargo" +args = ["${TOOLCHAIN}", "run", "--target", "${RUST_TARGET}"] + +[tasks.clean] +description = "Clean the project" +command = "cargo" +args = ["${TOOLCHAIN}", "clean", "--target", "${RUST_TARGET}"] + +[tasks.test] +description = "Run the tests" +command = "cargo" +args = ["${TOOLCHAIN}", "test", "--target", "${RUST_TARGET}"] + +[tasks.doc] +description = "Generate documentation" +command = "cargo" +args = ["${TOOLCHAIN}", "doc", "--target", "${RUST_TARGET}"] + +[tasks.fmt] +description = "Format the code" +command = "cargo" +args = ["${TOOLCHAIN}", "fmt"] + +[tasks.fmt-check] +description = "Check code format" +command = "cargo" +args = ["${TOOLCHAIN}", "fmt", "--check"] + +[tasks.run-release] +description = "Run the project in release mode" +command = "cargo" +args = ["${TOOLCHAIN}", "run", "--target", "${RUST_TARGET}", "--release"] + +[tasks.build-release] +description = "Build the project in release mode" +command = "cargo" +args = ["${TOOLCHAIN}", "build", "--target", "${RUST_TARGET}", "--release"] + +[tasks.test-release] +description = "Test the project in release mode" +command = "cargo" +args = ["${TOOLCHAIN}", "test", "--target", "${RUST_TARGET}", "--release"] + +[tasks.install] +description = "Install the project" +command = "cargo" +args = ["${TOOLCHAIN}", "install", "--target", "${RUST_TARGET}", "--path", ".", "--no-track", "--root", "${INSTALL_DIR}", "--force"] diff --git a/user/apps/test_namespace/src/main.rs b/user/apps/test_namespace/src/main.rs new file mode 100644 index 000000000..3a2b1d33d --- /dev/null +++ b/user/apps/test_namespace/src/main.rs @@ -0,0 +1,38 @@ +extern crate nix; +use nix::sched::{self, CloneFlags}; +use nix::sys::wait::{waitpid, WaitStatus}; +use nix::unistd::{self, fork, ForkResult}; +use std::process; + +fn main() { + let clone_flags = CloneFlags::CLONE_NEWPID | CloneFlags::CLONE_NEWNS; + + println!("Parent process. PID: {}", unistd::getpid()); + unsafe { + match fork() { + Ok(ForkResult::Parent { child }) => { + println!("Parent process. Child PID: {}", child); + match waitpid(child, None) { + Ok(WaitStatus::Exited(pid, status)) => { + println!("Child {} exited with status: {}", pid, status); + } + Ok(_) => println!("Child process did not exit normally."), + Err(e) => println!("Error waiting for child process: {:?}", e), + } + } + Ok(ForkResult::Child) => { + // 使用 unshare 创建新的命名空间 + println!("Child process. PID: {}", unistd::getpid()); + if let Err(e) = sched::unshare(clone_flags) { + println!("Failed to unshare: {:?}", e); + process::exit(1); + } + println!("Child process. PID: {}", unistd::getpid()); + } + Err(err) => { + println!("Fork failed: {:?}", err); + process::exit(1); + } + } + } +} diff --git a/user/apps/test_overlayfs/.gitignore b/user/apps/test_overlayfs/.gitignore new file mode 100644 index 000000000..614b14a88 --- /dev/null +++ b/user/apps/test_overlayfs/.gitignore @@ -0,0 +1 @@ +test_ovrlayfs \ No newline at end of file diff --git a/user/apps/test_overlayfs/Makefile b/user/apps/test_overlayfs/Makefile new file mode 100644 index 000000000..6be387383 --- /dev/null +++ b/user/apps/test_overlayfs/Makefile @@ -0,0 +1,20 @@ +ifeq ($(ARCH), x86_64) + CROSS_COMPILE=x86_64-linux-musl- +else ifeq ($(ARCH), riscv64) + CROSS_COMPILE=riscv64-linux-musl- +endif + +CC=$(CROSS_COMPILE)gcc + +.PHONY: all +all: main.c + $(CC) -static -o test_overlayfs main.c + +.PHONY: install clean +install: all + mv test_overlayfs $(DADK_CURRENT_BUILD_DIR)/test_overlayfs + +clean: + rm test_overlayfs *.o + +fmt: diff --git a/user/apps/test_overlayfs/main.c b/user/apps/test_overlayfs/main.c new file mode 100644 index 000000000..180371d87 --- /dev/null +++ b/user/apps/test_overlayfs/main.c @@ -0,0 +1,92 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +// #define LOWERDIR "/tmp/overlayfs/lower" +// #define UPPERDIR "/tmp/overlayfs/upper" +// #define WORKDIR "/tmp/overlayfs/work" +// #define MERGEDDIR "/tmp/overlayfs/merged" + +// void create_directories() +// { +// mkdir(LOWERDIR, 0755); +// mkdir(UPPERDIR, 0755); +// mkdir(WORKDIR, 0755); +// mkdir(MERGEDDIR, 0755); +// } +#define TMPDIR "/tmp" +#define OVERLAYFSDIR "/tmp/overlayfs" +#define LOWERDIR "/tmp/overlayfs/lower" +#define UPPERDIR "/tmp/overlayfs/upper" +#define WORKDIR "/tmp/overlayfs/work" +#define MERGEDDIR "/tmp/overlayfs/merged" + +void create_directories() +{ + mkdir(TMPDIR, 0755); + mkdir(OVERLAYFSDIR, 0755); + mkdir(LOWERDIR, 0755); + mkdir(UPPERDIR, 0755); + mkdir(WORKDIR, 0755); + mkdir(MERGEDDIR, 0755); + printf("step1 : success\n"); +} + +void create_lower_file() +{ + char filepath[256]; + snprintf(filepath, sizeof(filepath), "%s/lowerfile.txt", LOWERDIR); + + int fd = open(filepath, O_CREAT | O_WRONLY, 0644); + if (fd < 0) + { + perror("Failed to create file in lowerdir"); + exit(EXIT_FAILURE); + } + write(fd, "This is a lower layer file.\n", 28); + close(fd); + printf("step2 : success\n"); +} + +void mount_overlayfs() +{ + char options[1024]; + snprintf(options, sizeof(options), + "lowerdir=%s,upperdir=%s,workdir=%s", + LOWERDIR, UPPERDIR, WORKDIR); + + if (mount("overlay", MERGEDDIR, "overlay", 0, options) != 0) + { + perror("Mount failed"); + exit(EXIT_FAILURE); + } + printf("OverlayFS mounted successfully.\n"); + printf("step3 : success\n"); +} + +void create_directory_in_merged() +{ + char dirpath[256]; + snprintf(dirpath, sizeof(dirpath), "%s/newdir", UPPERDIR); + + if (mkdir(dirpath, 0755) != 0) + { + perror("Failed to create directory in merged dir"); + exit(EXIT_FAILURE); + } + printf("Directory created in merged: %s\n", dirpath); + printf("step4 : success\n"); +} + +int main() +{ + create_directories(); + mount_overlayfs(); + create_directory_in_merged(); + return 0; +} \ No newline at end of file diff --git a/user/dadk/config/test_namespace.dadk b/user/dadk/config/test_namespace.dadk new file mode 100644 index 000000000..29f1afc62 --- /dev/null +++ b/user/dadk/config/test_namespace.dadk @@ -0,0 +1,25 @@ +{ + "name": "test_namespace", + "version": "0.1.0", + "description": "test namespace", + "rust_target": null, + "task_type": { + "BuildFromSource": { + "Local": { + "path": "apps/test_namespace" + } + } + }, + "depends": [], + "build": { + "build_command": "make install" + }, + "install": { + "in_dragonos_path": "/bin" + }, + "clean": { + "clean_command": "make clean" + }, + "envs": [], + "target_arch": ["x86_64"] +} \ No newline at end of file diff --git a/user/dadk/config/test_overlayfs.dadk b/user/dadk/config/test_overlayfs.dadk new file mode 100644 index 000000000..03592a6c8 --- /dev/null +++ b/user/dadk/config/test_overlayfs.dadk @@ -0,0 +1,25 @@ +{ + "name": "test_overlayfs", + "version": "0.1.0", + "description": "test overlayfs", + "rust_target": null, + "task_type": { + "BuildFromSource": { + "Local": { + "path": "apps/test_overlayfs" + } + } + }, + "depends": [], + "build": { + "build_command": "make install" + }, + "install": { + "in_dragonos_path": "/bin" + }, + "clean": { + "clean_command": "make clean" + }, + "envs": [], + "target_arch": ["x86_64"] +} \ No newline at end of file From 7b0ef10895108a0de5ff5ef3d2f93f40cf2e33a5 Mon Sep 17 00:00:00 2001 From: LoGin Date: Thu, 31 Oct 2024 01:05:34 +0800 Subject: [PATCH 17/30] ci: change rust src to crates-io-index while using GitHub workflow (#1023) Signed-off-by: longjin --- .github/workflows/makefile.yml | 16 ++++++++-------- tools/BUILD_CONTAINER_VERSION | 2 +- tools/docker-entrypoint.sh | 16 ++++++++++++++++ 3 files changed, 25 insertions(+), 9 deletions(-) diff --git a/.github/workflows/makefile.yml b/.github/workflows/makefile.yml index cfb550834..6a1612e4a 100644 --- a/.github/workflows/makefile.yml +++ b/.github/workflows/makefile.yml @@ -12,14 +12,14 @@ jobs: name: Format check ${{ matrix.arch }} runs-on: ubuntu-latest continue-on-error: true - container: dragonos/dragonos-dev:v1.4 + container: dragonos/dragonos-dev:v1.5 strategy: matrix: arch: [x86_64, riscv64] steps: - - run: echo "Running in dragonos/dragonos-dev:v1.4" + - run: echo "Running in dragonos/dragonos-dev:v1.5" - uses: actions/checkout@v3 - name: Format check @@ -35,14 +35,14 @@ jobs: name: Kernel static test ${{ matrix.arch }} runs-on: ubuntu-latest continue-on-error: true - container: dragonos/dragonos-dev:v1.4 + container: dragonos/dragonos-dev:v1.5 strategy: matrix: arch: [x86_64, riscv64] steps: - - run: echo "Running in dragonos/dragonos-dev:v1.4" + - run: echo "Running in dragonos/dragonos-dev:v1.5" - uses: actions/checkout@v3 @@ -56,10 +56,10 @@ jobs: build-x86_64: runs-on: ubuntu-latest - container: dragonos/dragonos-dev:v1.4 + container: dragonos/dragonos-dev:v1.5 steps: - - run: echo "Running in dragonos/dragonos-dev:v1.4" + - run: echo "Running in dragonos/dragonos-dev:v1.5" - uses: actions/checkout@v3 - name: build the DragonOS @@ -78,10 +78,10 @@ jobs: build-riscv64: runs-on: ubuntu-latest - container: dragonos/dragonos-dev:v1.4 + container: dragonos/dragonos-dev:v1.5 steps: - - run: echo "Running in dragonos/dragonos-dev:v1.4" + - run: echo "Running in dragonos/dragonos-dev:v1.5" - uses: actions/checkout@v3 with: diff --git a/tools/BUILD_CONTAINER_VERSION b/tools/BUILD_CONTAINER_VERSION index 64c411b81..59b6ef75f 100644 --- a/tools/BUILD_CONTAINER_VERSION +++ b/tools/BUILD_CONTAINER_VERSION @@ -1 +1 @@ -v1.4 \ No newline at end of file +v1.5 \ No newline at end of file diff --git a/tools/docker-entrypoint.sh b/tools/docker-entrypoint.sh index 5fc44481d..ea8ccd655 100644 --- a/tools/docker-entrypoint.sh +++ b/tools/docker-entrypoint.sh @@ -1,3 +1,19 @@ #!/bin/bash +CONFIG_FILE=~/.cargo/config.toml + +change_rust_src_to_official() { +echo -e "[source.crates-io] \n \ +registry = \"sparse+https://index.crates.io/\" \n \ +[net] \n \ +git-fetch-with-cli = true \n \ +" > $CONFIG_FILE +} + +# Check if the GITHUB_WORKFLOW environment variable is set and not empty +if [ -n "$GITHUB_ACTION" ]; then + change_rust_src_to_official +fi + + exec "$@" From 2e7e4cecb0ebb42d4c417997a7ebc4891f2826e6 Mon Sep 17 00:00:00 2001 From: MemoryShore <1353318529@qq.com> Date: Thu, 31 Oct 2024 13:48:23 +0800 Subject: [PATCH 18/30] Update NovaShell to cb835e03e4 (#1022) --- user/dadk/config/nova_shell-0.1.0.dadk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/user/dadk/config/nova_shell-0.1.0.dadk b/user/dadk/config/nova_shell-0.1.0.dadk index b3cd99ec4..490a1250b 100644 --- a/user/dadk/config/nova_shell-0.1.0.dadk +++ b/user/dadk/config/nova_shell-0.1.0.dadk @@ -6,7 +6,7 @@ "BuildFromSource": { "Git": { "url": "https://git.mirrors.dragonos.org.cn/DragonOS-Community/NovaShell.git", - "revision": "7bb802ad1e" + "revision": "cb835e03e4" } } }, From c7ad61d495d56fe2db26fa454699d1720b8115fb Mon Sep 17 00:00:00 2001 From: LoGin Date: Thu, 31 Oct 2024 22:14:45 +0800 Subject: [PATCH 19/30] =?UTF-8?q?fix:=20=E5=88=A0=E9=99=A4=E5=9C=A8=20#949?= =?UTF-8?q?=20=E4=B8=AD=E6=84=8F=E5=A4=96=E6=B7=BB=E5=8A=A0=E7=9A=84?= =?UTF-8?q?=E6=97=A7=E7=89=88ida=20(#1027)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit https://github.com/DragonOS-Community/DragonOS/pull/949 Signed-off-by: longjin --- kernel/src/libs/ida/src/lib.rs | 55 ---------------------------------- 1 file changed, 55 deletions(-) delete mode 100644 kernel/src/libs/ida/src/lib.rs diff --git a/kernel/src/libs/ida/src/lib.rs b/kernel/src/libs/ida/src/lib.rs deleted file mode 100644 index ac25d557c..000000000 --- a/kernel/src/libs/ida/src/lib.rs +++ /dev/null @@ -1,55 +0,0 @@ -#![no_std] -#![feature(core_intrinsics)] -#![allow(clippy::needless_return)] - -use core::intrinsics::unlikely; -use core::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; - -/// id分配器 -/// -/// TODO: 当前只是为了简单实现功能，将来这里应使用类似linux的ida的方式去实现 -#[derive(Debug)] -pub struct IdAllocator { - current_id: AtomicUsize, - max_id: usize, - dead: AtomicBool, -} - -impl IdAllocator { - /// 创建一个新的id分配器 - pub const fn new(initial_id: usize, max_id: usize) -> Self { - Self { - current_id: AtomicUsize::new(initial_id), - max_id, - dead: AtomicBool::new(false), - } - } - - /// 分配一个新的id - /// - /// ## 返回 - /// - /// 如果分配成功，返回Some(id)，否则返回None - pub fn alloc(&self) -> Option { - if unlikely(self.dead.load(Ordering::SeqCst)) { - return None; - } - - let ret = self.current_id.fetch_add(1, Ordering::SeqCst); - // 如果id溢出，panic - if ret == self.max_id { - self.dead.store(true, Ordering::SeqCst); - return None; - } - - return Some(ret); - } - - pub fn free(&self, _id: usize) { - // todo: free - } - - pub fn get_max_id(&self) -> usize { - self.max_id - } -} From 01b8a76cdc5088e524b9f3c45c5ff827e88293d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=81=AB=E8=8A=B1?= Date: Tue, 5 Nov 2024 16:47:44 +0800 Subject: [PATCH 20/30] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E9=98=BB=E5=A1=9E?= =?UTF-8?q?=E9=98=85=E8=AF=BBeventfd=E6=97=B6=E6=97=A0=E6=B3=95=E8=A2=AB?= =?UTF-8?q?=E4=B8=AD=E6=96=AD=E7=9A=84bug=20(#1030)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: sparkzky --- kernel/src/filesystem/eventfd.rs | 37 ++++++++++++++++++++------------ 1 file changed, 23 insertions(+), 14 deletions(-) diff --git a/kernel/src/filesystem/eventfd.rs b/kernel/src/filesystem/eventfd.rs index 4d0d9a76e..9143697a9 100644 --- a/kernel/src/filesystem/eventfd.rs +++ b/kernel/src/filesystem/eventfd.rs @@ -5,6 +5,7 @@ use crate::libs::spinlock::{SpinLock, SpinLockGuard}; use crate::libs::wait_queue::WaitQueue; use crate::net::event_poll::{EPollEventType, EPollItem, EventPoll, KernelIoctlData}; use crate::process::ProcessManager; +use crate::sched::SchedMode; use crate::syscall::Syscall; use alloc::collections::LinkedList; use alloc::string::String; @@ -76,6 +77,11 @@ impl EventFdInode { Err(SystemError::ENOENT) } + + fn readable(&self) -> bool { + let count = self.eventfd.lock().count; + return count > 0; + } } impl IndexNode for EventFdInode { @@ -104,26 +110,29 @@ impl IndexNode for EventFdInode { _offset: usize, len: usize, buf: &mut [u8], - data: SpinLockGuard, + data_guard: SpinLockGuard, ) -> Result { + let data = data_guard.clone(); + drop(data_guard); if len < 8 { return Err(SystemError::EINVAL); } - let mut val = loop { - let val = self.eventfd.lock().count; - if val != 0 { - break val; - } - if self - .eventfd - .lock() - .flags - .contains(EventFdFlags::EFD_NONBLOCK) - { + let mut lock_efd = self.eventfd.lock(); + while lock_efd.count == 0 { + if lock_efd.flags.contains(EventFdFlags::EFD_NONBLOCK) { + drop(lock_efd); return Err(SystemError::EAGAIN_OR_EWOULDBLOCK); } - self.wait_queue.sleep(); - }; + + drop(lock_efd); + let r = wq_wait_event_interruptible!(self.wait_queue, self.readable(), {}); + if r.is_err() { + return Err(SystemError::ERESTARTSYS); + } + + lock_efd = self.eventfd.lock(); + } + let mut val = lock_efd.count; let mut eventfd = self.eventfd.lock(); if eventfd.flags.contains(EventFdFlags::EFD_SEMAPHORE) { From 2b180c4fa54ece554e59e0c7d783ee38b4e7a555 Mon Sep 17 00:00:00 2001 From: Samuka007 Date: Fri, 8 Nov 2024 14:54:56 +0800 Subject: [PATCH 21/30] debug and add socket option level print --- kernel/src/arch/x86_64/syscall/mod.rs | 13 ++-- kernel/src/net/socket/inet/stream/mod.rs | 80 ++++++++++++++++++++++++ kernel/src/net/syscall.rs | 2 +- 3 files changed, 88 insertions(+), 7 deletions(-) diff --git a/kernel/src/arch/x86_64/syscall/mod.rs b/kernel/src/arch/x86_64/syscall/mod.rs index ef9ee600b..054aba3b2 100644 --- a/kernel/src/arch/x86_64/syscall/mod.rs +++ b/kernel/src/arch/x86_64/syscall/mod.rs @@ -104,8 +104,9 @@ pub extern "sysv64" fn syscall_handler(frame: &mut TrapFrame) { ]; mfence(); let pid = ProcessManager::current_pcb().pid(); - let mut show = (syscall_num != SYS_SCHED) && (pid.data() >= 7); - // let mut show = true; + let mut show = + (syscall_num != SYS_SCHED) && (pid.data() >= 7); + // false; let to_print = SysCall::try_from(syscall_num); if let Ok(to_print) = to_print { @@ -115,21 +116,21 @@ pub extern "sysv64" fn syscall_handler(frame: &mut TrapFrame) { // show &= false; } SYS_RECVFROM | SYS_SENDTO | SYS_SENDMSG | SYS_RECVMSG => { - show &= false; + // show &= false; } SYS_SOCKET | SYS_GETSOCKNAME | SYS_GETPEERNAME | SYS_SOCKETPAIR | SYS_SETSOCKOPT | SYS_GETSOCKOPT => { - show &= false; + // show &= false; } SYS_OPEN | SYS_OPENAT | SYS_CREAT | SYS_CLOSE => { - show &= false; + // show &= false; } SYS_READ | SYS_WRITE | SYS_READV | SYS_WRITEV | SYS_PREAD64 | SYS_PWRITE64 | SYS_PREADV | SYS_PWRITEV | SYS_PREADV2 => { show &= false; } _ => { - show &= false; + // show &= false; } } if show { diff --git a/kernel/src/net/socket/inet/stream/mod.rs b/kernel/src/net/socket/inet/stream/mod.rs index c56419eaf..0c2b9ff99 100644 --- a/kernel/src/net/socket/inet/stream/mod.rs +++ b/kernel/src/net/socket/inet/stream/mod.rs @@ -334,6 +334,86 @@ impl Socket for TcpSocket { }) .unwrap_or(Ok(())) } + + fn set_option(&self, level: PSOL, name: usize, val: &[u8]) -> Result<(), SystemError> { + if level != PSOL::TCP { + // return Err(EINVAL); + log::debug!("TcpSocket::set_option: not TCP"); + return Ok(()); + } + use option::Options::{self, *}; + let option_name = Options::try_from(name as i32)?; + log::debug!("TCP Option: {:?}, value = {:?}", option_name, val); + match option_name { + NoDelay => { + let nagle_enabled = val[0] != 0; + let mut writer = self.inner.write(); + let inner = writer.take().expect("Tcp Inner is None"); + match inner { + Inner::Established(established) => { + established.with_mut(|socket| { + socket.set_nagle_enabled(nagle_enabled); + }); + writer.replace(Inner::Established(established)); + } + _ => { + writer.replace(inner); + return Err(EINVAL); + } + } + }, + KeepIntvl => { + if val.len() == 4 { + let mut writer = self.inner.write(); + let inner = writer.take().expect("Tcp Inner is None"); + match inner { + Inner::Established(established) => { + let interval = u32::from_ne_bytes([val[0], val[1], val[2], val[3]]); + established.with_mut(|socket| { + socket.set_keep_alive(Some(smoltcp::time::Duration::from_secs(interval as u64))); + }); + writer.replace(Inner::Established(established)); + } + _ => { + writer.replace(inner); + return Err(EINVAL); + } + } + } else { + return Err(EINVAL); + } + }, + KeepCnt => { + // if val.len() == 4 { + // let mut writer = self.inner.write(); + // let inner = writer.take().expect("Tcp Inner is None"); + // match inner { + // Inner::Established(established) => { + // let count = u32::from_ne_bytes([val[0], val[1], val[2], val[3]]); + // established.with_mut(|socket| { + // socket.set_keep_alive_count(count); + // }); + // writer.replace(Inner::Established(established)); + // } + // _ => { + // writer.replace(inner); + // return Err(EINVAL); + // } + // } + // } else { + // return Err(EINVAL); + // } + }, + KeepIdle => { + + }, + _ => { + log::debug!("TcpSocket::set_option: not supported"); + // return Err(ENOPROTOOPT); + } + } + Ok(()) + } } impl InetSocket for TcpSocket { diff --git a/kernel/src/net/syscall.rs b/kernel/src/net/syscall.rs index 8d2cf8aa8..bea15e209 100644 --- a/kernel/src/net/syscall.rs +++ b/kernel/src/net/syscall.rs @@ -116,7 +116,7 @@ impl Syscall { let socket: Arc = ProcessManager::current_pcb() .get_socket(fd as i32) .ok_or(SystemError::EBADF)?; - debug!("setsockopt: level={:?}", level); + debug!("setsockopt: level = {:?} ", sol); return socket.set_option(sol, optname, optval).map(|_| 0); } From 5d54e74768dafa01ea3f78af84da46cb9e07f0fd Mon Sep 17 00:00:00 2001 From: LoGin Date: Sun, 10 Nov 2024 20:04:23 +0800 Subject: [PATCH 22/30] fix: add gitignore to test_namespace (#1038) --- user/apps/test_namespace/.gitignore | 1 + 1 file changed, 1 insertion(+) create mode 100644 user/apps/test_namespace/.gitignore diff --git a/user/apps/test_namespace/.gitignore b/user/apps/test_namespace/.gitignore new file mode 100644 index 000000000..ea8c4bf7f --- /dev/null +++ b/user/apps/test_namespace/.gitignore @@ -0,0 +1 @@ +/target From f5c732d8dc4803803c9a9461b36164e85215e69f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9B=BE=E4=BF=8A?= <110876916+ZZJJWarth@users.noreply.github.com> Date: Sun, 10 Nov 2024 23:46:56 +0800 Subject: [PATCH 23/30] =?UTF-8?q?fix(video):=20=E5=A2=9E=E5=8A=A0=E4=BA=86?= =?UTF-8?q?=E5=AF=B9frame=20buffer=E6=93=8D=E4=BD=9C=E7=9A=84=E5=AE=89?= =?UTF-8?q?=E5=85=A8=E6=A3=80=E6=9F=A5=20(#1034)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit fix(video): 增加了对frame buffer操作的安全检查 --- kernel/src/driver/video/fbdev/base/mod.rs | 117 ++++++---------- .../driver/video/fbdev/base/render_helper.rs | 128 +++++++++++++++++- 2 files changed, 171 insertions(+), 74 deletions(-) diff --git a/kernel/src/driver/video/fbdev/base/mod.rs b/kernel/src/driver/video/fbdev/base/mod.rs index fb9921a68..0f3b6081c 100644 --- a/kernel/src/driver/video/fbdev/base/mod.rs +++ b/kernel/src/driver/video/fbdev/base/mod.rs @@ -1,4 +1,5 @@ use alloc::{string::String, sync::Arc, vec::Vec}; +use render_helper::{FrameP, FramePointerStatus}; use system_error::SystemError; use crate::{ @@ -69,22 +70,20 @@ pub trait FrameBuffer: FrameBufferInfo + FrameBufferOps + Device { let bit_per_pixel = self.current_fb_var().bits_per_pixel; // 计算图像在帧缓冲中的起始位 - let mut bitstart = (y * self.current_fb_fix().line_length * 8) + (x * bit_per_pixel); + let bitstart = (y * self.current_fb_fix().line_length * 8) + (x * bit_per_pixel); let start_index = bitstart & (32 - 1); let pitch_index = (self.current_fb_fix().line_length & (byte_per_pixel - 1)) * 8; - // 位转字节 - bitstart /= 8; - - // 对齐到像素字节大小 - bitstart &= !(byte_per_pixel - 1); - - let dst1 = boot_param.screen_info.lfb_virt_base; - if dst1.is_none() { + let dst2 = boot_param.screen_info.lfb_virt_base; + if dst2.is_none() { return; } - let mut dst1 = dst1.unwrap(); - dst1 += VirtAddr::new(bitstart as usize); - + let mut safe_pointer = FrameP::new( + self.current_fb_var().yres as usize, + self.current_fb_var().xres as usize, + self.current_fb_var().bits_per_pixel as usize, + dst2.unwrap(), + image, + ); let _ = self.fb_sync(); if image.depth == 1 { @@ -107,16 +106,9 @@ pub trait FrameBuffer: FrameBufferInfo + FrameBufferOps + Device { && image.width & (32 / bit_per_pixel - 1) == 0 && (8..=32).contains(&bit_per_pixel) { - unsafe { self.fast_imageblit(image, dst1, fg, bg) } + unsafe { self.fast_imageblit(image, &mut safe_pointer, fg, bg) } } else { - self.slow_imageblit( - image, - dst1, - fg, - bg, - bitstart / 4, - self.current_fb_fix().line_length, - ) + self.slow_imageblit(image, &mut safe_pointer, fg, bg) } } else { todo!("color image blit todo"); @@ -129,7 +121,7 @@ pub trait FrameBuffer: FrameBufferInfo + FrameBufferOps + Device { /// 要求 image->width 可以被像素或 dword (ppw) 整除。 /// 要求 fix->line_length 可以被 4 整除。 /// 扫描线的开始和结束都是 dword 对齐的。 - unsafe fn fast_imageblit(&self, image: &FbImage, mut dst1: VirtAddr, fg: u32, bg: u32) { + unsafe fn fast_imageblit(&self, image: &FbImage, dst1: &mut FrameP, fg: u32, bg: u32) { let bpp = self.current_fb_var().bits_per_pixel; let mut fgx = fg; let mut bgx = bg; @@ -161,13 +153,12 @@ pub trait FrameBuffer: FrameBufferInfo + FrameBufferOps + Device { color_tab[idx] = (*val & eorx) ^ bgx; } - let mut dst; let mut shift; let mut src; let mut offset = 0; let mut j = 0; + let mut count = 0; for _ in (0..image.height).rev() { - dst = dst1.as_ptr::(); shift = 8; src = offset; match ppw { @@ -175,10 +166,8 @@ pub trait FrameBuffer: FrameBufferInfo + FrameBufferOps + Device { // 8bpp j = k; while j >= 2 { - *dst = color_tab[(image.data[src] as usize >> 4) & bitmask]; - dst = dst.add(1); - *dst = color_tab[(image.data[src] as usize) & bitmask]; - dst = dst.add(1); + dst1.write(color_tab[(image.data[src] as usize >> 4) & bitmask]); + dst1.write(color_tab[(image.data[src] as usize) & bitmask]); j -= 2; src += 1; } @@ -187,14 +176,10 @@ pub trait FrameBuffer: FrameBufferInfo + FrameBufferOps + Device { // 16bpp j = k; while j >= 4 { - *dst = color_tab[(image.data[src] as usize >> 6) & bitmask]; - dst = dst.add(1); - *dst = color_tab[(image.data[src] as usize >> 4) & bitmask]; - dst = dst.add(1); - *dst = color_tab[(image.data[src] as usize >> 2) & bitmask]; - dst = dst.add(1); - *dst = color_tab[(image.data[src] as usize) & bitmask]; - dst = dst.add(1); + dst1.write(color_tab[(image.data[src] as usize >> 6) & bitmask]); + dst1.write(color_tab[(image.data[src] as usize >> 4) & bitmask]); + dst1.write(color_tab[(image.data[src] as usize >> 2) & bitmask]); + dst1.write(color_tab[(image.data[src] as usize) & bitmask]); src += 1; j -= 4; } @@ -203,22 +188,14 @@ pub trait FrameBuffer: FrameBufferInfo + FrameBufferOps + Device { // 32 bpp j = k; while j >= 8 { - *dst = color_tab[(image.data[src] as usize >> 7) & bitmask]; - dst = dst.add(1); - *dst = color_tab[(image.data[src] as usize >> 6) & bitmask]; - dst = dst.add(1); - *dst = color_tab[(image.data[src] as usize >> 5) & bitmask]; - dst = dst.add(1); - *dst = color_tab[(image.data[src] as usize >> 4) & bitmask]; - dst = dst.add(1); - *dst = color_tab[(image.data[src] as usize >> 3) & bitmask]; - dst = dst.add(1); - *dst = color_tab[(image.data[src] as usize >> 2) & bitmask]; - dst = dst.add(1); - *dst = color_tab[(image.data[src] as usize >> 1) & bitmask]; - dst = dst.add(1); - *dst = color_tab[(image.data[src] as usize) & bitmask]; - dst = dst.add(1); + dst1.write(color_tab[(image.data[src] as usize >> 7) & bitmask]); + dst1.write(color_tab[(image.data[src] as usize >> 6) & bitmask]); + dst1.write(color_tab[(image.data[src] as usize >> 5) & bitmask]); + dst1.write(color_tab[(image.data[src] as usize >> 4) & bitmask]); + dst1.write(color_tab[(image.data[src] as usize >> 3) & bitmask]); + dst1.write(color_tab[(image.data[src] as usize >> 2) & bitmask]); + dst1.write(color_tab[(image.data[src] as usize >> 1) & bitmask]); + dst1.write(color_tab[(image.data[src] as usize) & bitmask]); src += 1; j -= 8; } @@ -233,8 +210,7 @@ pub trait FrameBuffer: FrameBufferInfo + FrameBufferOps + Device { */ while j != 0 { shift -= ppw; - *dst = color_tab[(image.data[src] as usize >> shift) & bitmask]; - dst = dst.add(1); + dst1.write(color_tab[(image.data[src] as usize >> shift) & bitmask]); if shift == 0 { shift = 8; src += 1; @@ -242,22 +218,15 @@ pub trait FrameBuffer: FrameBufferInfo + FrameBufferOps + Device { j -= 1; } - dst1 += VirtAddr::new(self.current_fb_fix().line_length as usize); + count += 1; + dst1.move_with_offset(self.current_fb_fix().line_length * count); offset += spitch as usize; } } - fn slow_imageblit( - &self, - _image: &FbImage, - _dst1: VirtAddr, - _fg: u32, - _bg: u32, - _start_index: u32, - _pitch_index: u32, - ) { - let mut dst = _dst1.as_ptr::(); + fn slow_imageblit(&self, _image: &FbImage, safe_dst: &mut FrameP, _fg: u32, _bg: u32) { let mut count = 0; + let mut pt_status = FramePointerStatus::Normal; let iter = BitIter::new( _fg, _bg, @@ -268,17 +237,19 @@ pub trait FrameBuffer: FrameBufferInfo + FrameBufferOps + Device { _image.width, ); for (content, full) in iter { - unsafe { - *dst = content; - - dst = dst.add(1); + match pt_status { + FramePointerStatus::OutOfBuffer => { + return; + } + FramePointerStatus::OutOfScreen => {} + FramePointerStatus::Normal => { + pt_status = safe_dst.write(content); + } } - if full { count += 1; - dst = unsafe { - _dst1.as_ptr::().add((_pitch_index * count) as usize) as *mut u32 - }; + safe_dst.move_with_offset(self.current_fb_fix().line_length * count); + pt_status = FramePointerStatus::Normal; } } } diff --git a/kernel/src/driver/video/fbdev/base/render_helper.rs b/kernel/src/driver/video/fbdev/base/render_helper.rs index 0ce9f80a5..25b2d5cb3 100644 --- a/kernel/src/driver/video/fbdev/base/render_helper.rs +++ b/kernel/src/driver/video/fbdev/base/render_helper.rs @@ -1,4 +1,8 @@ -use core::slice::Iter; +use core::{ops::Add, slice::Iter}; + +use crate::mm::VirtAddr; + +use super::FbImage; pub struct BitIter<'a> { fgcolor: u32, @@ -170,3 +174,125 @@ impl PixelLineStatus { } } } + +/// # 结构功能 +/// 安全的FrameBufferPointer +/// 使用该结构体访问FrameBuffer可以防止超出FrameBuffer区域的访问 +/// 需要注意，使用该指针写入时，任何超出屏幕的写入都是无效的！即使仍然可以写入显存。 +/// 此外由于FbImage中的x和y变量采用u32类型，所以并未考虑左越界和上越界的安全性(即Image.x<0或Image.y<0的情况) +/// ## 成员 +/// +/// - "dst" : 显存base address，通常是0xffffa1003ff00000 +/// - "limit" : 显存区域上界，可以通过公式计算：limit = dst + 分辨率高*分辨率宽*每个像素的**字节**数。也就是说任何对于显存的访问应该限制在[dst,limit)中 +/// - "current" : 当前相对于start_offset的位移 +/// - "start_offset" : 如果你要渲染某个Image，你可能不是总是从屏幕左上角(0,0)开始渲染，你可能从某个offset开始 +/// - "start_xpos" : 表示你要渲染的Image的x位置的字节位置 +/// - "current_xpos" : 当前渲染的x位置的字节位置 +/// - "limit_xpos" : 最大的渲染x位置的字节位置。 例：假设系统的分辨率为640，位深为24，你需要渲染的Image的x坐标为200，那么start_xpos=200*3=600，current_xpos=200*3+当前行已经渲染像素数*3，limit_xpos=640*3 +#[derive(Debug)] +pub struct FrameP { + dst: VirtAddr, + limit: VirtAddr, + current: usize, + start_offset: usize, + start_xpos: usize, + current_xpos: usize, + limit_xpos: usize, +} + +impl FrameP { + pub fn new( + frame_height: usize, + frame_width: usize, + bitdepth: usize, + dst: VirtAddr, + image: &FbImage, + ) -> Self { + let byte_per_pixel = bitdepth / 8; + let limit = VirtAddr::new(frame_height * frame_width * byte_per_pixel) + dst; + Self { + dst, + limit, + current: 0, + start_offset: start_offset(image, bitdepth as u32, (frame_width * bitdepth / 8) as u32) + as usize, + start_xpos: image.x as usize * byte_per_pixel, + current_xpos: image.x as usize * byte_per_pixel, + limit_xpos: frame_width * byte_per_pixel, + } + } + + /// # 函数功能 + /// 写入某个数据并将指针增大 + pub fn write(&mut self, data: T) -> FramePointerStatus { + // 首先获取数据大小 + let size = size_of::(); + // 复制显存指针防止改变self的dst + let mut dst = self.dst; + + // 你最终要写入的位置实际上是[dst+start_offset+current,dst+start_offset+current+size),所以我们要确定你写入的位置是否超过limit + if self.dst.data() + self.current + self.start_offset + size > self.limit.data() { + return FramePointerStatus::OutOfBuffer; + } + // 我们也不希望你的x超出屏幕右边，超出屏幕右边的部分会被忽略掉，因为如果写入显存会导致内存风险 + else if self.current_xpos + size > self.limit_xpos { + return FramePointerStatus::OutOfScreen; + } + // 如果上面两个检查都通过了，我们就可以写入显存了 + else { + // 这里是写入位置的实际虚拟地址 + dst = dst.add(self.current + self.start_offset); + } + // 写入操作 + unsafe { + *dst.as_ptr::() = data; + } + // 写入后更新current和xpos + self.current += size; + self.current_xpos += size; + // 由于写入正常，我们返回正常的状态 + return FramePointerStatus::Normal; + } + + /// # 函数功能 + /// 移动指针**至**某个offset + /// todo: 当前函数应当只用于换行，否则可能会导致安全性问题，即offset应该是每行像素的开头 + pub fn move_with_offset(&mut self, offset: u32) { + self.current = offset as usize; + // let x_pos=self.current%self.limit_xpos; + // match x_pos{ + // n if n{ + // // send_to_default_serial8250_port(format!("Sended by function move_with_offset: Check if there is misusage of offset,the image.x is:{:?} while the xpos indicated by the offset is:{:?},current FP:{:?}\n",self.start_offset,x_pos,self).as_bytes()); + // } + // n if n>=self.limit_xpos=>{ + // // send_to_default_serial8250_port(format!("Sended by function move_with_offset: Check if there is misusage of offset,The offset:{:?} is so large that it would exceed the limit of frame buffer\n",offset).as_bytes()); + // } + // _=>{ + + // } + // } + self.current_xpos = self.start_xpos; + } +} + +pub enum FramePointerStatus { + /// 表示状态正常 + Normal, + /// 超出屏幕，一直到换行时才应该恢复到正常状态 + OutOfScreen, + /// 超出缓存，此时应当立即停止渲染 + OutOfBuffer, +} + +pub fn start_offset(image: &FbImage, bitdepth: u32, line_length: u32) -> u32 { + let x = image.x; + let y = image.y; + let mut bitstart = (y * line_length * 8) + (x * bitdepth); + let byte_per_pixel = core::mem::size_of::() as u32; + // 位转字节 + bitstart /= 8; + + // 对齐到像素字节大小 + bitstart &= !(byte_per_pixel - 1); + return bitstart; +} From 20c58101dda979c0df4ab3f76f21555464cdfeeb Mon Sep 17 00:00:00 2001 From: LoGin Date: Mon, 11 Nov 2024 11:38:57 +0800 Subject: [PATCH 24/30] =?UTF-8?q?fix:=20do=5Fsignal=E5=87=BD=E6=95=B0?= =?UTF-8?q?=E5=9C=A8=E5=A4=84=E7=90=86SIG=5FKILL=E6=97=B6,=E6=9C=89?= =?UTF-8?q?=E4=B8=80=E5=A4=84pcb=E5=BC=BA=E5=BC=95=E7=94=A8=E6=9C=AA?= =?UTF-8?q?=E5=BD=92=E9=9B=B6=E7=9A=84=E9=97=AE=E9=A2=98=20(#1039)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: longjin --- kernel/src/arch/x86_64/ipc/signal.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/kernel/src/arch/x86_64/ipc/signal.rs b/kernel/src/arch/x86_64/ipc/signal.rs index c9d0af99e..5bc2db64a 100644 --- a/kernel/src/arch/x86_64/ipc/signal.rs +++ b/kernel/src/arch/x86_64/ipc/signal.rs @@ -431,8 +431,6 @@ impl SignalArch for X86_64SignalArch { return; } - let pcb = ProcessManager::current_pcb(); - let mut sig_number: Signal; let mut info: Option; let mut sigaction: Sigaction; @@ -483,9 +481,13 @@ impl SignalArch for X86_64SignalArch { //避免死锁 drop(siginfo_mut_guard); drop(sig_guard); + drop(pcb); // 做完上面的检查后，开中断 CurrentIrqArch::interrupt_enable(); + + // 注意！由于handle_signal里面可能会退出进程， + // 因此这里需要检查清楚：上面所有的锁、arc指针都被释放了。否则会产生资源泄露的问题！ let res: Result = handle_signal(sig_number, &mut sigaction, &info.unwrap(), &oldset, frame); if res.is_err() { From 69715438f21c9a4da5198b92aee5c27aa8c44543 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9B=BE=E4=BF=8A?= <110876916+ZZJJWarth@users.noreply.github.com> Date: Mon, 11 Nov 2024 20:57:08 +0800 Subject: [PATCH 25/30] =?UTF-8?q?feat(pci):=20=E8=A7=A3=E5=86=B3=E4=BA=86p?= =?UTF-8?q?ci=E6=80=BB=E7=BA=BF=E5=AD=90=E7=B3=BB=E7=BB=9F=E4=B8=ADpci?= =?UTF-8?q?=E7=BB=93=E6=9E=84=E4=B8=80=E8=87=B4=E6=80=A7=E9=97=AE=E9=A2=98?= =?UTF-8?q?=EF=BC=8C=E5=A2=9E=E5=8A=A0=E4=BA=86pci=E8=AE=BE=E5=A4=87?= =?UTF-8?q?=E7=9A=84=E5=8F=AF=E8=AF=BB=E5=B1=9E=E6=80=A7=20(#1009)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --------- Co-authored-by: 黄铭涛 <1037827920@qq.com> --- kernel/src/driver/disk/ahci/mod.rs | 20 +-- kernel/src/driver/net/e1000e/e1000e.rs | 49 +++--- kernel/src/driver/pci/attr.rs | 161 +++++++++++++++++++- kernel/src/driver/pci/device.rs | 9 +- kernel/src/driver/pci/pci.rs | 177 ++++++++++------------ kernel/src/driver/pci/pci_irq.rs | 111 +++++++------- kernel/src/driver/pci/raw_device.rs | 30 +++- kernel/src/driver/pci/test/pt_device.rs | 28 +++- kernel/src/driver/virtio/transport.rs | 3 +- kernel/src/driver/virtio/transport_pci.rs | 25 +-- kernel/src/driver/virtio/virtio.rs | 19 +-- 11 files changed, 416 insertions(+), 216 deletions(-) diff --git a/kernel/src/driver/disk/ahci/mod.rs b/kernel/src/driver/disk/ahci/mod.rs index 6fd31d859..38e036d60 100644 --- a/kernel/src/driver/disk/ahci/mod.rs +++ b/kernel/src/driver/disk/ahci/mod.rs @@ -2,23 +2,22 @@ pub mod ahci_inode; pub mod ahcidisk; pub mod hba; - use crate::arch::MMArch; use crate::driver::base::block::manager::block_dev_manager; use crate::driver::block::cache::cached_block_device::BlockCache; use crate::driver::disk::ahci::ahcidisk::LockedAhciDisk; use crate::driver::pci::pci::{ - get_pci_device_structure_mut, PciDeviceStructure, PCI_DEVICE_LINKEDLIST, + get_pci_device_structure_mut, PciDeviceLinkedList, PciDeviceStructure, PCI_DEVICE_LINKEDLIST, }; +use alloc::sync::Arc; use crate::driver::disk::ahci::{ hba::HbaMem, hba::{HbaPort, HbaPortType}, }; -use crate::libs::rwlock::RwLockWriteGuard; use crate::libs::spinlock::{SpinLock, SpinLockGuard}; use crate::mm::{MemoryManagementArch, VirtAddr}; -use alloc::{boxed::Box, collections::LinkedList, vec::Vec}; +use alloc::{boxed::Box, vec::Vec}; use core::sync::atomic::compiler_fence; use log::debug; use system_error::SystemError; @@ -36,9 +35,9 @@ pub const HBA_PxIS_TFES: u32 = 1 << 30; /// @brief 寻找所有的ahci设备 /// @param list 链表的写锁 /// @return Result>, SystemError> 成功则返回包含所有ahci设备结构体的可变引用的链表，失败则返回err -fn ahci_device_search<'a>( - list: &'a mut RwLockWriteGuard<'_, LinkedList>>, -) -> Result>, SystemError> { +fn ahci_device_search( + list: &PciDeviceLinkedList, +) -> Result>, SystemError> { let result = get_pci_device_structure_mut(list, AHCI_CLASS, AHCI_SUBCLASS); if result.is_empty() { @@ -50,11 +49,11 @@ fn ahci_device_search<'a>( /// @brief: 初始化 ahci pub fn ahci_init() -> Result<(), SystemError> { - let mut list = PCI_DEVICE_LINKEDLIST.write(); - let ahci_device = ahci_device_search(&mut list)?; + let list = &*PCI_DEVICE_LINKEDLIST; + let ahci_device = ahci_device_search(list)?; for device in ahci_device { - let standard_device = device.as_standard_device_mut().unwrap(); + let standard_device = device.as_standard_device().unwrap(); standard_device.bar_ioremap(); // 对于每一个ahci控制器分配一块空间 let ahci_port_base_vaddr = @@ -62,6 +61,7 @@ pub fn ahci_init() -> Result<(), SystemError> { let virtaddr = standard_device .bar() .ok_or(SystemError::EACCES)? + .read() .get_bar(5) .or(Err(SystemError::EACCES))? .virtual_address() diff --git a/kernel/src/driver/net/e1000e/e1000e.rs b/kernel/src/driver/net/e1000e/e1000e.rs index a144b1386..0383c8ac9 100644 --- a/kernel/src/driver/net/e1000e/e1000e.rs +++ b/kernel/src/driver/net/e1000e/e1000e.rs @@ -1,16 +1,6 @@ // 参考手册: PCIe* GbE Controllers Open Source Software Developer’s Manual // Refernce: PCIe* GbE Controllers Open Source Software Developer’s Manual -use alloc::string::ToString; -use alloc::sync::Arc; -use alloc::vec::Vec; -use core::intrinsics::unlikely; -use core::mem::size_of; -use core::ptr::NonNull; -use core::slice::{from_raw_parts, from_raw_parts_mut}; -use core::sync::atomic::{compiler_fence, Ordering}; -use log::{debug, info}; - use super::e1000e_driver::e1000e_driver_init; use crate::driver::base::device::DeviceId; use crate::driver::net::dma::{dma_alloc, dma_dealloc}; @@ -21,6 +11,15 @@ use crate::driver::pci::pci::{ }; use crate::driver::pci::pci_irq::{IrqCommonMsg, IrqSpecificMsg, PciInterrupt, PciIrqMsg, IRQ}; use crate::exception::IrqNumber; +use alloc::string::ToString; +use alloc::sync::Arc; +use alloc::vec::Vec; +use core::intrinsics::unlikely; +use core::mem::size_of; +use core::ptr::NonNull; +use core::slice::{from_raw_parts, from_raw_parts_mut}; +use core::sync::atomic::{compiler_fence, Ordering}; +use log::{debug, info}; use crate::libs::volatile::{ReadOnly, Volatile, WriteOnly}; @@ -200,14 +199,14 @@ impl E1000EDevice { // init the device for PCI standard device struct #[allow(unused_assignments)] pub fn new( - device: &mut PciDeviceStructureGeneralDevice, + device: Arc, device_id: Arc, ) -> Result { // 从BAR0获取我们需要的寄存器 // Build registers sturcts from BAR0 device.bar_ioremap().unwrap()?; device.enable_master(); - let bar = device.bar().ok_or(E1000EPciError::BarGetFailed)?; + let bar = device.bar().ok_or(E1000EPciError::BarGetFailed)?.read(); let bar0 = bar.get_bar(0)?; let (address, size) = bar0 .memory_address_size() @@ -226,7 +225,7 @@ impl E1000EDevice { // 初始化msi中断 // initialize msi interupt let irq_vector = device.irq_vector_mut().unwrap(); - irq_vector.push(E1000E_RECV_VECTOR); + irq_vector.write().push(E1000E_RECV_VECTOR); device.irq_init(IRQ::PCI_IRQ_MSI).expect("IRQ Init Failed"); let msg = PciIrqMsg { irq_common_message: IrqCommonMsg::init_from( @@ -598,26 +597,32 @@ pub fn e1000e_init() { } pub fn e1000e_probe() -> Result { - let mut list = PCI_DEVICE_LINKEDLIST.write(); - let result = get_pci_device_structure_mut(&mut list, NETWORK_CLASS, ETHERNET_SUBCLASS); + let list = &*PCI_DEVICE_LINKEDLIST; + let result = get_pci_device_structure_mut(list, NETWORK_CLASS, ETHERNET_SUBCLASS); if result.is_empty() { return Ok(0); } for device in result { - let standard_device = device.as_standard_device_mut().unwrap(); - let header = &standard_device.common_header; - if header.vendor_id == 0x8086 { + let standard_device = device.as_standard_device().unwrap(); + if standard_device.common_header.vendor_id == 0x8086 { // intel - if E1000E_DEVICE_ID.contains(&header.device_id) { + if E1000E_DEVICE_ID.contains(&standard_device.common_header.device_id) { debug!( "Detected e1000e PCI device with device id {:#x}", - header.device_id + standard_device.common_header.device_id ); // todo: 根据pci的path来生成device id let e1000e = E1000EDevice::new( - standard_device, - DeviceId::new(None, Some(format!("e1000e_{}", header.device_id))).unwrap(), + standard_device.clone(), + DeviceId::new( + None, + Some(format!( + "e1000e_{}", + standard_device.common_header.device_id + )), + ) + .unwrap(), )?; e1000e_driver_init(e1000e); } diff --git a/kernel/src/driver/pci/attr.rs b/kernel/src/driver/pci/attr.rs index 6d59122be..081f7715e 100644 --- a/kernel/src/driver/pci/attr.rs +++ b/kernel/src/driver/pci/attr.rs @@ -13,7 +13,7 @@ use crate::{ }, }; -use super::device::PciDevice; +use super::{device::PciDevice, pci_irq::IrqType}; #[derive(Debug)] pub struct BasicPciReadOnlyAttrs; @@ -23,7 +23,16 @@ impl AttributeGroup for BasicPciReadOnlyAttrs { } fn attrs(&self) -> &[&'static dyn Attribute] { - &[&Vendor, &DeviceID, &SubsystemVendor, &SubsystemDevice] + &[ + &Vendor, + &DeviceID, + &SubsystemVendor, + &SubsystemDevice, + &Revision, + &Class, + &Irq, + &Modalias, + ] } fn is_visible( @@ -36,7 +45,7 @@ impl AttributeGroup for BasicPciReadOnlyAttrs { } #[derive(Debug)] -pub struct Vendor; +struct Vendor; impl Attribute for Vendor { fn mode(&self) -> ModeType { @@ -67,7 +76,7 @@ impl Attribute for Vendor { } #[derive(Debug)] -pub struct DeviceID; +struct DeviceID; impl Attribute for DeviceID { fn mode(&self) -> ModeType { @@ -98,7 +107,7 @@ impl Attribute for DeviceID { } #[derive(Debug)] -pub struct SubsystemVendor; +struct SubsystemVendor; impl Attribute for SubsystemVendor { fn mode(&self) -> ModeType { @@ -129,7 +138,7 @@ impl Attribute for SubsystemVendor { } #[derive(Debug)] -pub struct SubsystemDevice; +struct SubsystemDevice; impl Attribute for SubsystemDevice { fn mode(&self) -> ModeType { @@ -158,3 +167,143 @@ impl Attribute for SubsystemDevice { SysFSOpsSupport::ATTR_SHOW } } + +#[derive(Debug)] +struct Revision; + +impl Attribute for Revision { + fn mode(&self) -> ModeType { + SYSFS_ATTR_MODE_RO + } + + fn name(&self) -> &str { + "revision" + } + + fn show(&self, _kobj: Arc, _buf: &mut [u8]) -> Result { + let dev = _kobj + .cast::() + .map_err(|e: Arc| { + warn!("device:{:?} is not a pci device!", e); + SystemError::EINVAL + })?; + return sysfs_emit_str(_buf, &format!("0x{:02x}", dev.revision())); + } + + fn store(&self, _kobj: Arc, _buf: &[u8]) -> Result { + todo!() + } + + fn support(&self) -> SysFSOpsSupport { + SysFSOpsSupport::ATTR_SHOW + } +} + +#[derive(Debug)] +struct Class; + +impl Attribute for Class { + fn mode(&self) -> ModeType { + SYSFS_ATTR_MODE_RO + } + + fn name(&self) -> &str { + "class" + } + + fn show(&self, _kobj: Arc, _buf: &mut [u8]) -> Result { + let dev = _kobj + .cast::() + .map_err(|e: Arc| { + warn!("device:{:?} is not a pci device!", e); + SystemError::EINVAL + })?; + return sysfs_emit_str(_buf, &format!("0x{:06x}", dev.class_code())); + } + + fn store(&self, _kobj: Arc, _buf: &[u8]) -> Result { + todo!() + } + + fn support(&self) -> SysFSOpsSupport { + SysFSOpsSupport::ATTR_SHOW + } +} + +#[derive(Debug)] +struct Irq; + +impl Attribute for Irq { + fn mode(&self) -> ModeType { + SYSFS_ATTR_MODE_RO + } + + fn name(&self) -> &str { + "irq" + } + + fn show(&self, _kobj: Arc, _buf: &mut [u8]) -> Result { + let dev = _kobj + .cast::() + .map_err(|e: Arc| { + warn!("device:{:?} is not a pci device!", e); + SystemError::EINVAL + })?; + if let IrqType::Msi { .. } = *dev.irq_type().read() { + // 见https://code.dragonos.org.cn/xref/linux-6.1.9/drivers/pci/pci-sysfs.c#55 + return sysfs_emit_str(_buf, "todo:sry,msi device is unimplemented now"); + } + return sysfs_emit_str(_buf, &format!("{}", dev.irq_line())); + } + + fn store(&self, _kobj: Arc, _buf: &[u8]) -> Result { + todo!() + } + + fn support(&self) -> SysFSOpsSupport { + SysFSOpsSupport::ATTR_SHOW + } +} + +#[derive(Debug)] +struct Modalias; + +impl Attribute for Modalias { + fn mode(&self) -> ModeType { + SYSFS_ATTR_MODE_RO + } + + fn name(&self) -> &str { + "modalias" + } + + fn show(&self, _kobj: Arc, _buf: &mut [u8]) -> Result { + let dev = _kobj + .cast::() + .map_err(|e: Arc| { + warn!("device:{:?} is not a pci device!", e); + SystemError::EINVAL + })?; + return sysfs_emit_str( + _buf, + &format!( + "pci:v{:08X}d{:08X}sv{:08X}sd{:08X}bc{:02X}sc{:02X}i{:02X}", + dev.vendor(), + dev.device_id(), + dev.subsystem_vendor(), + dev.subsystem_device(), + dev.class_code(), + dev.subclass(), + dev.interface_code(), + ), + ); + } + + fn store(&self, _kobj: Arc, _buf: &[u8]) -> Result { + todo!() + } + + fn support(&self) -> SysFSOpsSupport { + SysFSOpsSupport::ATTR_SHOW + } +} diff --git a/kernel/src/driver/pci/device.rs b/kernel/src/driver/pci/device.rs index e93b25de8..22668e56b 100644 --- a/kernel/src/driver/pci/device.rs +++ b/kernel/src/driver/pci/device.rs @@ -14,13 +14,14 @@ use crate::{ }, filesystem::kernfs::KernFSInode, libs::{ - rwlock::RwLockWriteGuard, + rwlock::{RwLock, RwLockWriteGuard}, spinlock::{SpinLock, SpinLockGuard}, }, }; use super::{ dev_id::PciDeviceID, + pci_irq::IrqType, subsys::{pci_bus, pci_bus_device}, }; @@ -83,6 +84,12 @@ pub trait PciDevice: Device { fn device_id(&self) -> u16; fn subsystem_vendor(&self) -> u16; fn subsystem_device(&self) -> u16; + fn revision(&self) -> u8; + fn class_code(&self) -> u8; + fn irq_type(&self) -> &RwLock; + fn irq_line(&self) -> u8; + fn subclass(&self) -> u8; + fn interface_code(&self) -> u8; } /// #结构功能 diff --git a/kernel/src/driver/pci/pci.rs b/kernel/src/driver/pci/pci.rs index b339529f9..0a22fe6da 100644 --- a/kernel/src/driver/pci/pci.rs +++ b/kernel/src/driver/pci/pci.rs @@ -16,7 +16,7 @@ use crate::mm::mmio_buddy::{mmio_pool, MMIOSpaceGuard}; use crate::mm::VirtAddr; use alloc::string::String; -use alloc::sync::Arc; +use alloc::sync::{Arc, Weak}; use alloc::vec::Vec; use alloc::{boxed::Box, collections::LinkedList}; use bitflags::bitflags; @@ -68,7 +68,7 @@ impl Debug for PciAddr { /// 添加了读写锁的链表，存储PCI设备结构体 pub struct PciDeviceLinkedList { - list: RwLock>>, + list: RwLock>>, } impl PciDeviceLinkedList { @@ -80,12 +80,12 @@ impl PciDeviceLinkedList { } /// @brief 获取可读的linkedlist(读锁守卫) /// @return RwLockReadGuard>> 读锁守卫 - pub fn read(&self) -> RwLockReadGuard>> { + pub fn read(&self) -> RwLockReadGuard>> { self.list.read() } /// @brief 获取可写的linkedlist(写锁守卫) /// @return RwLockWriteGuard>> 写锁守卫 - pub fn write(&self) -> RwLockWriteGuard>> { + pub fn write(&self) -> RwLockWriteGuard>> { self.list.write() } /// @brief 获取链表中PCI结构体数目 @@ -95,7 +95,7 @@ impl PciDeviceLinkedList { list.len() } /// @brief 添加Pci设备结构体到链表中 - pub fn add(&self, device: Box) { + pub fn add(&self, device: Arc) { let mut list = self.list.write(); list.push_back(device); } @@ -113,15 +113,14 @@ impl PciDeviceLinkedList { /// ## 返回值 /// /// - 返回匹配的供应商ID的PCI设备结构的引用。 -pub fn get_pci_device_structures_mut_by_vendor_id<'a>( - list: &'a mut RwLockWriteGuard<'_, LinkedList>>, +pub fn get_pci_device_structures_mut_by_vendor_id( + list: &PciDeviceLinkedList, vendor_id: u16, -) -> Vec<&'a mut Box<(dyn PciDeviceStructure)>> { - let mut result = Vec::new(); - for box_pci_device_structure in list.iter_mut() { - let common_header = (*box_pci_device_structure).common_header(); - if common_header.vendor_id == vendor_id { - result.push(box_pci_device_structure); +) -> Vec> { + let mut result: Vec> = Vec::new(); + for box_pci_device_structure in list.write().iter() { + if box_pci_device_structure.common_header().vendor_id == vendor_id { + result.push(box_pci_device_structure.clone()); } } result @@ -140,16 +139,17 @@ pub fn get_pci_device_structures_mut_by_vendor_id<'a>( /// /// ## 返回值 /// - 包含链表中所有满足条件的PCI结构体的可变引用的容器。 -pub fn get_pci_device_structure_mut<'a>( - list: &'a mut RwLockWriteGuard<'_, LinkedList>>, +pub fn get_pci_device_structure_mut( + list: &PciDeviceLinkedList, class_code: u8, subclass: u8, -) -> Vec<&'a mut Box<(dyn PciDeviceStructure)>> { +) -> Vec> { let mut result = Vec::new(); - for box_pci_device_structure in list.iter_mut() { - let common_header = (*box_pci_device_structure).common_header(); - if (common_header.class_code == class_code) && (common_header.subclass == subclass) { - result.push(box_pci_device_structure); + for box_pci_device_structure in list.write().iter() { + if (box_pci_device_structure.common_header().class_code == class_code) + && (box_pci_device_structure.common_header().subclass == subclass) + { + result.push(box_pci_device_structure.clone()); } } result @@ -317,7 +317,7 @@ pub trait PciDeviceStructure: Send + Sync { fn header_type(&self) -> HeaderType; /// @brief 当其为standard设备时返回&Pci_Device_Structure_General_Device，其余情况返回None #[inline(always)] - fn as_standard_device(&self) -> Option<&PciDeviceStructureGeneralDevice> { + fn as_standard_device(&self) -> Option> { None } /// @brief 当其为pci to pci bridge设备时返回&Pci_Device_Structure_Pci_to_Pci_Bridge，其余情况返回None @@ -333,21 +333,14 @@ pub trait PciDeviceStructure: Send + Sync { /// @brief 获取Pci设备共有的common_header /// @return 返回其不可变引用 fn common_header(&self) -> &PciDeviceStructureHeader; - /// @brief 当其为standard设备时返回&mut Pci_Device_Structure_General_Device，其余情况返回None - #[inline(always)] - fn as_standard_device_mut(&mut self) -> Option<&mut PciDeviceStructureGeneralDevice> { - None - } /// @brief 当其为pci to pci bridge设备时返回&mut Pci_Device_Structure_Pci_to_Pci_Bridge，其余情况返回None #[inline(always)] - fn as_pci_to_pci_bridge_device_mut(&mut self) -> Option<&mut PciDeviceStructurePciToPciBridge> { + fn as_pci_to_pci_bridge_device_mut(&self) -> Option<&PciDeviceStructurePciToPciBridge> { None } /// @brief 当其为pci to cardbus bridge设备时返回&mut Pci_Device_Structure_Pci_to_Cardbus_Bridge，其余情况返回None #[inline(always)] - fn as_pci_to_carbus_bridge_device_mut( - &mut self, - ) -> Option<&mut PciDeviceStructurePciToCardbusBridge> { + fn as_pci_to_carbus_bridge_device_mut(&self) -> Option<&PciDeviceStructurePciToCardbusBridge> { None } /// @brief 返回迭代器，遍历capabilities @@ -358,14 +351,14 @@ pub trait PciDeviceStructure: Send + Sync { fn status_command(&self) -> (Status, Command) { let common_header = self.common_header(); let status = Status::from_bits_truncate(common_header.status); - let command = Command::from_bits_truncate(common_header.command); + let command = Command::from_bits_truncate(*common_header.command.read()); (status, command) } /// @brief 设置Command寄存器的值 - fn set_command(&mut self, command: Command) { + fn set_command(&self, command: Command) { let common_header = self.common_header_mut(); let command = command.bits(); - common_header.command = command; + *common_header.command.write() = command; pci_root_0().write_config( common_header.bus_device_function, STATUS_COMMAND_OFFSET.into(), @@ -374,22 +367,22 @@ pub trait PciDeviceStructure: Send + Sync { } /// @brief 获取Pci设备共有的common_header /// @return 返回其可变引用 - fn common_header_mut(&mut self) -> &mut PciDeviceStructureHeader; + fn common_header_mut(&self) -> &PciDeviceStructureHeader; /// @brief 读取standard设备的bar寄存器，映射后将结果加入结构体的standard_device_bar变量 /// @return 只有standard设备才返回成功或者错误，其余返回None #[inline(always)] - fn bar_ioremap(&mut self) -> Option> { + fn bar_ioremap(&self) -> Option> { None } /// @brief 获取PCI设备的bar寄存器的引用 /// @return #[inline(always)] - fn bar(&mut self) -> Option<&PciStandardDeviceBar> { + fn bar(&self) -> Option<&RwLock> { None } /// @brief 通过设置该pci设备的command - fn enable_master(&mut self) { + fn enable_master(&self) { self.set_command(Command::IO_SPACE | Command::MEMORY_SPACE | Command::BUS_MASTER); } /// @brief 寻找设备的msix空间的offset @@ -411,21 +404,21 @@ pub trait PciDeviceStructure: Send + Sync { None } /// @brief 返回结构体中的irq_type的可变引用 - fn irq_type_mut(&mut self) -> Option<&mut IrqType>; + fn irq_type_mut(&self) -> Option<&RwLock>; /// @brief 返回结构体中的irq_vector的可变引用 - fn irq_vector_mut(&mut self) -> Option<&mut Vec>; + fn irq_vector_mut(&self) -> Option<&RwLock>>; } /// Pci_Device_Structure_Header PCI设备结构体共有的头部 -#[derive(Clone, Debug)] +#[derive(Debug)] pub struct PciDeviceStructureHeader { // ==== busdevicefunction变量表示该结构体所处的位置 pub bus_device_function: BusDeviceFunction, pub vendor_id: u16, // 供应商ID 0xffff是一个无效值，在读取访问不存在的设备的配置空间寄存器时返回 pub device_id: u16, // 设备ID，标志特定设备 - pub command: u16, // 提供对设备生成和响应pci周期的能力的控制 向该寄存器写入0时，设备与pci总线断开除配置空间访问以外的所有连接 - pub status: u16, // 用于记录pci总线相关时间的状态信息寄存器 - pub revision_id: u8, // 修订ID，指定特定设备的修订标志符 + pub command: RwLock, // 提供对设备生成和响应pci周期的能力的控制 向该寄存器写入0时，设备与pci总线断开除配置空间访问以外的所有连接 + pub status: u16, // 用于记录pci总线相关时间的状态信息寄存器 + pub revision_id: u8, // 修订ID，指定特定设备的修订标志符 pub prog_if: u8, // 编程接口字节，一个只读寄存器，指定设备具有的寄存器级别的编程接口（如果有的话） pub subclass: u8, // 子类。指定设备执行的特定功能的只读寄存器 pub class_code: u8, // 类代码，一个只读寄存器，指定设备执行的功能类型 @@ -440,14 +433,14 @@ pub struct PciDeviceStructureHeader { } /// Pci_Device_Structure_General_Device PCI标准设备结构体 -#[derive(Clone, Debug)] +#[derive(Debug)] pub struct PciDeviceStructureGeneralDevice { pub common_header: PciDeviceStructureHeader, // 中断结构体，包括legacy,msi,msix三种情况 - pub irq_type: IrqType, + pub irq_type: RwLock, // 使用的中断号的vec集合 - pub irq_vector: Vec, - pub standard_device_bar: PciStandardDeviceBar, + pub irq_vector: RwLock>, + pub standard_device_bar: RwLock, pub cardbus_cis_pointer: u32, // 指向卡信息结构，供在 CardBus 和 PCI 之间共享芯片的设备使用。 pub subsystem_vendor_id: u16, pub subsystem_id: u16, @@ -460,6 +453,7 @@ pub struct PciDeviceStructureGeneralDevice { pub interrupt_pin: u8, // 指定设备使用的中断引脚。其中值为0x1INTA#、0x2INTB#、0x3INTC#、0x4INTD#，0x0表示设备不使用中断引脚。 pub min_grant: u8, // 一个只读寄存器，用于指定设备所需的突发周期长度（以 1/4 微秒为单位）（假设时钟速率为 33 MHz） pub max_latency: u8, // 一个只读寄存器，指定设备需要多长时间访问一次 PCI 总线（以 1/4 微秒为单位）。 + pub self_ptr: RwLock>, } impl PciDeviceStructure for PciDeviceStructureGeneralDevice { #[inline(always)] @@ -467,20 +461,16 @@ impl PciDeviceStructure for PciDeviceStructureGeneralDevice { HeaderType::Standard } #[inline(always)] - fn as_standard_device(&self) -> Option<&PciDeviceStructureGeneralDevice> { - Some(self) - } - #[inline(always)] - fn as_standard_device_mut(&mut self) -> Option<&mut PciDeviceStructureGeneralDevice> { - Some(self) + fn as_standard_device(&self) -> Option> { + self.self_ptr.read().upgrade() } #[inline(always)] fn common_header(&self) -> &PciDeviceStructureHeader { &self.common_header } #[inline(always)] - fn common_header_mut(&mut self) -> &mut PciDeviceStructureHeader { - &mut self.common_header + fn common_header_mut(&self) -> &PciDeviceStructureHeader { + &self.common_header } fn capabilities(&self) -> Option { Some(CapabilityIterator { @@ -488,37 +478,37 @@ impl PciDeviceStructure for PciDeviceStructureGeneralDevice { next_capability_offset: Some(self.capabilities_pointer), }) } - fn bar_ioremap(&mut self) -> Option> { + fn bar_ioremap(&self) -> Option> { let common_header = &self.common_header; match pci_bar_init(common_header.bus_device_function) { Ok(bar) => { - self.standard_device_bar = bar; + *self.standard_device_bar.write() = bar; Some(Ok(0)) } Err(e) => Some(Err(e)), } } - fn bar(&mut self) -> Option<&PciStandardDeviceBar> { + fn bar(&self) -> Option<&RwLock> { Some(&self.standard_device_bar) } #[inline(always)] - fn irq_type_mut(&mut self) -> Option<&mut IrqType> { - Some(&mut self.irq_type) + fn irq_type_mut(&self) -> Option<&RwLock> { + Some(&self.irq_type) } #[inline(always)] - fn irq_vector_mut(&mut self) -> Option<&mut Vec> { - Some(&mut self.irq_vector) + fn irq_vector_mut(&self) -> Option<&RwLock>> { + Some(&self.irq_vector) } } /// Pci_Device_Structure_Pci_to_Pci_Bridge pci-to-pci桥设备结构体 -#[derive(Clone, Debug)] +#[derive(Debug)] pub struct PciDeviceStructurePciToPciBridge { pub common_header: PciDeviceStructureHeader, // 中断结构体，包括legacy,msi,msix三种情况 - pub irq_type: IrqType, + pub irq_type: RwLock, // 使用的中断号的vec集合 - pub irq_vector: Vec, + pub irq_vector: RwLock>, pub bar0: u32, pub bar1: u32, pub primary_bus_number: u8, @@ -554,7 +544,7 @@ impl PciDeviceStructure for PciDeviceStructurePciToPciBridge { Some(self) } #[inline(always)] - fn as_pci_to_pci_bridge_device_mut(&mut self) -> Option<&mut PciDeviceStructurePciToPciBridge> { + fn as_pci_to_pci_bridge_device_mut(&self) -> Option<&PciDeviceStructurePciToPciBridge> { Some(self) } #[inline(always)] @@ -562,20 +552,20 @@ impl PciDeviceStructure for PciDeviceStructurePciToPciBridge { &self.common_header } #[inline(always)] - fn common_header_mut(&mut self) -> &mut PciDeviceStructureHeader { - &mut self.common_header + fn common_header_mut(&self) -> &PciDeviceStructureHeader { + &self.common_header } #[inline(always)] - fn irq_type_mut(&mut self) -> Option<&mut IrqType> { - Some(&mut self.irq_type) + fn irq_type_mut(&self) -> Option<&RwLock> { + Some(&self.irq_type) } #[inline(always)] - fn irq_vector_mut(&mut self) -> Option<&mut Vec> { - Some(&mut self.irq_vector) + fn irq_vector_mut(&self) -> Option<&RwLock>> { + Some(&self.irq_vector) } } /// Pci_Device_Structure_Pci_to_Cardbus_Bridge Pci_to_Cardbus桥设备结构体 -#[derive(Clone, Debug)] +#[derive(Debug)] pub struct PciDeviceStructurePciToCardbusBridge { pub common_header: PciDeviceStructureHeader, pub cardbus_socket_ex_ca_base_address: u32, @@ -611,9 +601,7 @@ impl PciDeviceStructure for PciDeviceStructurePciToCardbusBridge { Some(self) } #[inline(always)] - fn as_pci_to_carbus_bridge_device_mut( - &mut self, - ) -> Option<&mut PciDeviceStructurePciToCardbusBridge> { + fn as_pci_to_carbus_bridge_device_mut(&self) -> Option<&PciDeviceStructurePciToCardbusBridge> { Some(self) } #[inline(always)] @@ -621,15 +609,15 @@ impl PciDeviceStructure for PciDeviceStructurePciToCardbusBridge { &self.common_header } #[inline(always)] - fn common_header_mut(&mut self) -> &mut PciDeviceStructureHeader { - &mut self.common_header + fn common_header_mut(&self) -> &PciDeviceStructureHeader { + &self.common_header } #[inline(always)] - fn irq_type_mut(&mut self) -> Option<&mut IrqType> { + fn irq_type_mut(&self) -> Option<&RwLock> { None } #[inline(always)] - fn irq_vector_mut(&mut self) -> Option<&mut Vec> { + fn irq_vector_mut(&self) -> Option<&RwLock>> { None } } @@ -684,14 +672,14 @@ pub fn capabilities_offset(bus_device_function: BusDeviceFunction) -> Option fn pci_read_header( bus_device_function: BusDeviceFunction, add_to_list: bool, -) -> Result, PciError> { +) -> Result, PciError> { // 先读取公共header let result = pci_root_0().read_config(bus_device_function, 0x00); let vendor_id = result as u16; let device_id = (result >> 16) as u16; let result = pci_root_0().read_config(bus_device_function, 0x04); - let command = result as u16; + let command = RwLock::new(result as u16); let status = (result >> 16) as u16; let result = pci_root_0().read_config(bus_device_function, 0x08); @@ -727,22 +715,22 @@ fn pci_read_header( HeaderType::Standard => { let general_device: PciDeviceStructureGeneralDevice = pci_read_general_device_header(header, &bus_device_function); - let box_general_device = Box::new(general_device.clone()); - let box_general_device_clone = box_general_device.clone(); + let box_general_device = Arc::new(general_device); + *box_general_device.self_ptr.write() = Arc::downgrade(&box_general_device); if add_to_list { - PCI_DEVICE_LINKEDLIST.add(box_general_device); + PCI_DEVICE_LINKEDLIST.add(box_general_device.clone()); //这里实际上不应该使用clone，因为raw是用于sysfs的结构，但是实际上pci设备是在PCI_DEVICE_LINKEDLIST链表上的， //这就导致sysfs呈现的对pci设备的操控接口实际上操控的是pci设备描述符是一个副本 //但是无奈这里没有使用Arc //todo：修改pci设备描述符在静态链表中存在的方式，并修改这里的clone操作 - let raw = PciGeneralDevice::from(&general_device); + let raw = PciGeneralDevice::from(box_general_device.clone()); let _ = pci_device_manager().device_add(Arc::new(raw)); } - Ok(box_general_device_clone) + Ok(box_general_device) } HeaderType::PciPciBridge => { let pci_to_pci_bridge = pci_read_pci_to_pci_bridge_header(header, &bus_device_function); - let box_pci_to_pci_bridge = Box::new(pci_to_pci_bridge); + let box_pci_to_pci_bridge = Arc::new(pci_to_pci_bridge); let box_pci_to_pci_bridge_clone = box_pci_to_pci_bridge.clone(); if add_to_list { PCI_DEVICE_LINKEDLIST.add(box_pci_to_pci_bridge); @@ -752,7 +740,7 @@ fn pci_read_header( HeaderType::PciCardbusBridge => { let pci_cardbus_bridge = pci_read_pci_to_cardbus_bridge_header(header, &bus_device_function); - let box_pci_cardbus_bridge = Box::new(pci_cardbus_bridge); + let box_pci_cardbus_bridge = Arc::new(pci_cardbus_bridge); let box_pci_cardbus_bridge_clone = box_pci_cardbus_bridge.clone(); if add_to_list { PCI_DEVICE_LINKEDLIST.add(box_pci_cardbus_bridge); @@ -772,7 +760,7 @@ fn pci_read_general_device_header( common_header: PciDeviceStructureHeader, bus_device_function: &BusDeviceFunction, ) -> PciDeviceStructureGeneralDevice { - let standard_device_bar = PciStandardDeviceBar::default(); + let standard_device_bar = RwLock::new(PciStandardDeviceBar::default()); let cardbus_cis_pointer = pci_root_0().read_config(*bus_device_function, 0x28); let result = pci_root_0().read_config(*bus_device_function, 0x2c); @@ -795,8 +783,8 @@ fn pci_read_general_device_header( let max_latency = (result >> 24) as u8; PciDeviceStructureGeneralDevice { common_header, - irq_type: IrqType::Unused, - irq_vector: Vec::new(), + irq_type: RwLock::new(IrqType::Unused), + irq_vector: RwLock::new(Vec::new()), standard_device_bar, cardbus_cis_pointer, subsystem_vendor_id, @@ -810,6 +798,7 @@ fn pci_read_general_device_header( interrupt_pin, min_grant, max_latency, + self_ptr: RwLock::new(Weak::new()), } } @@ -865,8 +854,8 @@ fn pci_read_pci_to_pci_bridge_header( let bridge_control = (result >> 16) as u16; PciDeviceStructurePciToPciBridge { common_header, - irq_type: IrqType::Unused, - irq_vector: Vec::new(), + irq_type: RwLock::new(IrqType::Unused), + irq_vector: RwLock::new(Vec::new()), bar0, bar1, primary_bus_number, diff --git a/kernel/src/driver/pci/pci_irq.rs b/kernel/src/driver/pci/pci_irq.rs index ea11a25b2..f4ab543ac 100644 --- a/kernel/src/driver/pci/pci_irq.rs +++ b/kernel/src/driver/pci/pci_irq.rs @@ -155,7 +155,7 @@ pub trait PciInterrupt: PciDeviceStructure { /// @param self PCI设备的可变引用 /// @param flag 选择的中断类型（支持多个选择），如PCI_IRQ_ALL_TYPES表示所有中断类型均可，让系统按顺序进行选择 /// @return Option 失败返回None，成功则返回对应中断类型 - fn irq_init(&mut self, flag: IRQ) -> Option { + fn irq_init(&self, flag: IRQ) -> Option { // MSIX中断优先 if flag.contains(IRQ::PCI_IRQ_MSIX) { if let Some(cap_offset) = self.msix_capability_offset() { @@ -174,7 +174,7 @@ pub trait PciInterrupt: PciDeviceStructure { ); let pending_table_bar = (data & 0x07) as u8; let pending_table_offset = data & (!0x07); - *self.irq_type_mut()? = IrqType::Msix { + *self.irq_type_mut()?.write() = IrqType::Msix { msix_table_bar, msix_table_offset, pending_table_bar, @@ -201,7 +201,7 @@ pub trait PciInterrupt: PciDeviceStructure { let maskable = (message_control & 0x0100) != 0; let address_64 = (message_control & 0x0080) != 0; let irq_max_num = (1 << (((message_control & 0x000e) >> 1) + 1)) as u16; - *self.irq_type_mut()? = IrqType::Msi { + *self.irq_type_mut()?.write() = IrqType::Msi { address_64, maskable, irq_max_num, @@ -217,7 +217,7 @@ pub trait PciInterrupt: PciDeviceStructure { } // 最后选择legacy# if flag.contains(IRQ::PCI_IRQ_LEGACY) { - *self.irq_type_mut()? = IrqType::Legacy; + *self.irq_type_mut()?.write() = IrqType::Legacy; return Some(IrqType::Legacy); } None @@ -226,9 +226,9 @@ pub trait PciInterrupt: PciDeviceStructure { /// @brief 启动/关闭设备中断 /// @param self PCI设备的可变引用 /// @param enable 开启/关闭 - fn irq_enable(&mut self, enable: bool) -> Result { + fn irq_enable(&self, enable: bool) -> Result { if let Some(irq_type) = self.irq_type_mut() { - match *irq_type { + match *irq_type.read() { IrqType::Msix { .. } => { return self.msix_enable(enable); } @@ -248,9 +248,9 @@ pub trait PciInterrupt: PciDeviceStructure { /// @brief 启动/关闭设备MSIX中断 /// @param self PCI设备的可变引用 /// @param enable 开启/关闭 - fn msix_enable(&mut self, enable: bool) -> Result { + fn msix_enable(&self, enable: bool) -> Result { if let Some(irq_type) = self.irq_type_mut() { - match *irq_type { + match *irq_type.read() { IrqType::Msix { cap_offset, .. } => { let mut message = pci_root_0() .read_config(self.common_header().bus_device_function, cap_offset.into()); @@ -279,9 +279,9 @@ pub trait PciInterrupt: PciDeviceStructure { /// @brief 启动/关闭设备MSI中断 /// @param self PCI设备的可变引用 /// @param enable 开启/关闭 - fn msi_enable(&mut self, enable: bool) -> Result { + fn msi_enable(&self, enable: bool) -> Result { if let Some(irq_type) = self.irq_type_mut() { - match *irq_type { + match *irq_type.read() { IrqType::Msi { cap_offset, .. } => { let mut message = pci_root_0() .read_config(self.common_header().bus_device_function, cap_offset.into()); @@ -315,9 +315,9 @@ pub trait PciInterrupt: PciDeviceStructure { /// @param self PCI设备的可变引用 /// @param msg PCI设备install中断时需要传递的共同参数 /// @return 一切正常返回Ok(0),有错误返回对应错误原因 - fn irq_install(&mut self, msg: PciIrqMsg) -> Result { + fn irq_install(&self, msg: PciIrqMsg) -> Result { if let Some(irq_vector) = self.irq_vector_mut() { - if msg.irq_common_message.irq_index as usize > irq_vector.len() { + if msg.irq_common_message.irq_index as usize > irq_vector.read().len() { return Err(PciError::PciIrqError(PciIrqError::InvalidIrqIndex( msg.irq_common_message.irq_index, ))); @@ -325,7 +325,7 @@ pub trait PciInterrupt: PciDeviceStructure { } self.irq_enable(false)?; //中断设置更改前先关闭对应PCI设备的中断 if let Some(irq_type) = self.irq_type_mut() { - match *irq_type { + match *irq_type.read() { IrqType::Msix { .. } => { return self.msix_install(msg); } @@ -346,9 +346,9 @@ pub trait PciInterrupt: PciDeviceStructure { /// @param self PCI设备的可变引用 /// @param msg PCI设备install中断时需要传递的共同参数 /// @return 一切正常返回Ok(0),有错误返回对应错误原因 - fn msi_install(&mut self, msg: PciIrqMsg) -> Result { + fn msi_install(&self, msg: PciIrqMsg) -> Result { if let Some(irq_type) = self.irq_type_mut() { - match *irq_type { + match *irq_type.read() { IrqType::Msi { address_64, irq_max_num, @@ -356,11 +356,11 @@ pub trait PciInterrupt: PciDeviceStructure { .. } => { // 注意：MSI中断分配的中断号必须连续且大小为2的倍数 - if self.irq_vector_mut().unwrap().len() > irq_max_num as usize { + if self.irq_vector_mut().unwrap().read().len() > irq_max_num as usize { return Err(PciError::PciIrqError(PciIrqError::DeviceIrqOverflow)); } - let irq_num = - self.irq_vector_mut().unwrap()[msg.irq_common_message.irq_index as usize]; + let irq_num = self.irq_vector_mut().unwrap().read() + [msg.irq_common_message.irq_index as usize]; let irq_num = IrqNumber::new(irq_num.into()); let common_msg = &msg.irq_common_message; @@ -441,7 +441,7 @@ pub trait PciInterrupt: PciDeviceStructure { cap_offset.into(), ); let message_control = (data >> 16) as u16; - match self.irq_vector_mut().unwrap().len() { + match self.irq_vector_mut().unwrap().read().len() { 1 => { let temp = message_control & (!0x0070); pci_root_0().write_config( @@ -511,20 +511,20 @@ pub trait PciInterrupt: PciDeviceStructure { /// @param self PCI设备的可变引用 /// @param msg PCI设备install中断时需要传递的共同参数 /// @return 一切正常返回Ok(0),有错误返回对应错误原因 - fn msix_install(&mut self, msg: PciIrqMsg) -> Result { + fn msix_install(&self, msg: PciIrqMsg) -> Result { if let Some(irq_type) = self.irq_type_mut() { - match *irq_type { + match *irq_type.read() { IrqType::Msix { irq_max_num, msix_table_bar, msix_table_offset, .. } => { - if self.irq_vector_mut().unwrap().len() > irq_max_num as usize { + if self.irq_vector_mut().unwrap().read().len() > irq_max_num as usize { return Err(PciError::PciIrqError(PciIrqError::DeviceIrqOverflow)); } - let irq_num = - self.irq_vector_mut().unwrap()[msg.irq_common_message.irq_index as usize]; + let irq_num = self.irq_vector_mut().unwrap().read() + [msg.irq_common_message.irq_index as usize]; let common_msg = &msg.irq_common_message; @@ -571,7 +571,8 @@ pub trait PciInterrupt: PciDeviceStructure { //写入Message Data和Message Address let pcistandardbar = self .bar() - .ok_or(PciError::PciIrqError(PciIrqError::PciBarNotInited))?; + .ok_or(PciError::PciIrqError(PciIrqError::PciBarNotInited))? + .read(); let msix_bar = pcistandardbar.get_bar(msix_table_bar)?; let vaddr: crate::mm::VirtAddr = msix_bar .virtual_address() @@ -603,7 +604,7 @@ pub trait PciInterrupt: PciDeviceStructure { fn irq_uninstall(&mut self) -> Result { self.irq_enable(false)?; //中断设置更改前先关闭对应PCI设备的中断 if let Some(irq_type) = self.irq_type_mut() { - match *irq_type { + match *irq_type.read() { IrqType::Msix { .. } => { return self.msix_uninstall(); } @@ -622,15 +623,15 @@ pub trait PciInterrupt: PciDeviceStructure { } /// @brief 进行PCI设备中断的卸载（MSI） /// @param self PCI设备的可变引用 - fn msi_uninstall(&mut self) -> Result { + fn msi_uninstall(&self) -> Result { if let Some(irq_type) = self.irq_type_mut() { - match *irq_type { + match *irq_type.read() { IrqType::Msi { address_64, cap_offset, .. } => { - for vector in self.irq_vector_mut().unwrap() { + for vector in self.irq_vector_mut().unwrap().read().iter() { let irq = IrqNumber::new((*vector).into()); irq_manager().free_irq(irq, None); } @@ -670,9 +671,9 @@ pub trait PciInterrupt: PciDeviceStructure { } /// @brief 进行PCI设备中断的卸载(MSIX) /// @param self PCI设备的可变引用 - fn msix_uninstall(&mut self) -> Result { + fn msix_uninstall(&self) -> Result { if let Some(irq_type) = self.irq_type_mut() { - match *irq_type { + match *irq_type.read() { IrqType::Msix { irq_max_num, cap_offset, @@ -680,7 +681,7 @@ pub trait PciInterrupt: PciDeviceStructure { msix_table_offset, .. } => { - for vector in self.irq_vector_mut().unwrap() { + for vector in self.irq_vector_mut().unwrap().read().iter() { let irq = IrqNumber::new((*vector).into()); irq_manager().free_irq(irq, None); } @@ -692,7 +693,8 @@ pub trait PciInterrupt: PciDeviceStructure { let pcistandardbar = self .bar() .ok_or(PciError::PciIrqError(PciIrqError::PciBarNotInited)) - .unwrap(); + .unwrap() + .read(); let msix_bar = pcistandardbar.get_bar(msix_table_bar).unwrap(); for index in 0..irq_max_num { let vaddr = msix_bar @@ -726,7 +728,7 @@ pub trait PciInterrupt: PciDeviceStructure { /// @param irq_index 中断的位置（在vec中的index和安装的index相同） fn irq_mask(&mut self, irq_index: u16) -> Result { if let Some(irq_type) = self.irq_type_mut() { - match *irq_type { + match *irq_type.read() { IrqType::Msix { .. } => { return self.msix_mask(irq_index); } @@ -746,9 +748,9 @@ pub trait PciInterrupt: PciDeviceStructure { /// @brief 屏蔽相应位置的中断(MSI) /// @param self PCI设备的可变引用 /// @param irq_index 中断的位置（在vec中的index和安装的index相同） - fn msi_mask(&mut self, irq_index: u16) -> Result { + fn msi_mask(&self, irq_index: u16) -> Result { if let Some(irq_type) = self.irq_type_mut() { - match *irq_type { + match *irq_type.read() { IrqType::Msi { maskable, address_64, @@ -804,9 +806,9 @@ pub trait PciInterrupt: PciDeviceStructure { /// @brief 屏蔽相应位置的中断(MSIX) /// @param self PCI设备的可变引用 /// @param irq_index 中断的位置（在vec中的index和安装的index相同） - fn msix_mask(&mut self, irq_index: u16) -> Result { + fn msix_mask(&self, irq_index: u16) -> Result { if let Some(irq_type) = self.irq_type_mut() { - match *irq_type { + match *irq_type.read() { IrqType::Msix { irq_max_num, msix_table_bar, @@ -821,7 +823,8 @@ pub trait PciInterrupt: PciDeviceStructure { let pcistandardbar = self .bar() .ok_or(PciError::PciIrqError(PciIrqError::PciBarNotInited)) - .unwrap(); + .unwrap() + .read(); let msix_bar = pcistandardbar.get_bar(msix_table_bar).unwrap(); let vaddr = msix_bar.virtual_address().unwrap() + msix_table_offset as usize @@ -845,9 +848,9 @@ pub trait PciInterrupt: PciDeviceStructure { /// @brief 解除屏蔽相应位置的中断 /// @param self PCI设备的可变引用 /// @param irq_index 中断的位置（在vec中的index和安装的index相同） - fn irq_unmask(&mut self, irq_index: u16) -> Result { + fn irq_unmask(&self, irq_index: u16) -> Result { if let Some(irq_type) = self.irq_type_mut() { - match *irq_type { + match *irq_type.read() { IrqType::Msix { .. } => { return self.msix_unmask(irq_index); } @@ -867,9 +870,9 @@ pub trait PciInterrupt: PciDeviceStructure { /// @brief 解除屏蔽相应位置的中断（MSI） /// @param self PCI设备的可变引用 /// @param irq_index 中断的位置（在vec中的index和安装的index相同） - fn msi_unmask(&mut self, irq_index: u16) -> Result { + fn msi_unmask(&self, irq_index: u16) -> Result { if let Some(irq_type) = self.irq_type_mut() { - match *irq_type { + match *irq_type.read() { IrqType::Msi { maskable, address_64, @@ -924,9 +927,9 @@ pub trait PciInterrupt: PciDeviceStructure { /// @brief 解除屏蔽相应位置的中断(MSIX) /// @param self PCI设备的可变引用 /// @param irq_index 中断的位置（在vec中的index和安装的index相同） - fn msix_unmask(&mut self, irq_index: u16) -> Result { + fn msix_unmask(&self, irq_index: u16) -> Result { if let Some(irq_type) = self.irq_type_mut() { - match *irq_type { + match *irq_type.read() { IrqType::Msix { irq_max_num, msix_table_bar, @@ -941,7 +944,8 @@ pub trait PciInterrupt: PciDeviceStructure { let pcistandardbar = self .bar() .ok_or(PciError::PciIrqError(PciIrqError::PciBarNotInited)) - .unwrap(); + .unwrap() + .read(); let msix_bar = pcistandardbar.get_bar(msix_table_bar).unwrap(); let vaddr = msix_bar.virtual_address().unwrap() + msix_table_offset as usize @@ -966,9 +970,9 @@ pub trait PciInterrupt: PciDeviceStructure { /// @param self PCI设备的可变引用 /// @param irq_index 中断的位置（在vec中的index和安装的index相同） /// @return 是否在挂起过程中产生中断（异常情况也返回false） - fn irq_check_pending(&mut self, irq_index: u16) -> Result { + fn irq_check_pending(&self, irq_index: u16) -> Result { if let Some(irq_type) = self.irq_type_mut() { - match *irq_type { + match *irq_type.read() { IrqType::Msix { .. } => { return self.msix_check_pending(irq_index); } @@ -989,9 +993,9 @@ pub trait PciInterrupt: PciDeviceStructure { /// @param self PCI设备的可变引用 /// @param irq_index 中断的位置（在vec中的index和安装的index相同） /// @return 是否在挂起过程中产生中断（异常情况也返回false） - fn msi_check_pending(&mut self, irq_index: u16) -> Result { + fn msi_check_pending(&self, irq_index: u16) -> Result { if let Some(irq_type) = self.irq_type_mut() { - match *irq_type { + match *irq_type.read() { IrqType::Msi { maskable, address_64, @@ -1038,9 +1042,9 @@ pub trait PciInterrupt: PciDeviceStructure { /// @param self PCI设备的可变引用 /// @param irq_index 中断的位置（在vec中的index和安装的index相同） /// @return 是否在挂起过程中产生中断（异常情况也返回false） - fn msix_check_pending(&mut self, irq_index: u16) -> Result { + fn msix_check_pending(&self, irq_index: u16) -> Result { if let Some(irq_type) = self.irq_type_mut() { - match *irq_type { + match *irq_type.read() { IrqType::Msix { irq_max_num, pending_table_bar, @@ -1055,7 +1059,8 @@ pub trait PciInterrupt: PciDeviceStructure { let pcistandardbar = self .bar() .ok_or(PciError::PciIrqError(PciIrqError::PciBarNotInited)) - .unwrap(); + .unwrap() + .read(); let pending_bar = pcistandardbar.get_bar(pending_table_bar).unwrap(); let vaddr = pending_bar.virtual_address().unwrap() + pending_table_offset as usize diff --git a/kernel/src/driver/pci/raw_device.rs b/kernel/src/driver/pci/raw_device.rs index a93478c03..fa00d5244 100644 --- a/kernel/src/driver/pci/raw_device.rs +++ b/kernel/src/driver/pci/raw_device.rs @@ -37,9 +37,9 @@ struct InnerPciGeneralDevice { device_common: DeviceCommonData, } -impl From<&PciDeviceStructureGeneralDevice> for PciGeneralDevice { - fn from(value: &PciDeviceStructureGeneralDevice) -> Self { - let value = Arc::new(value.clone()); +impl From> for PciGeneralDevice { + fn from(value: Arc) -> Self { + // let value = Arc::new(value.clone()); let name: String = value.common_header.bus_device_function.into(); let kobj_state = LockedKObjectState::new(None); let dev_id = PciDeviceID::dummpy(); @@ -80,6 +80,30 @@ impl PciDevice for PciGeneralDevice { fn subsystem_device(&self) -> u16 { self.header.subsystem_id } + + fn class_code(&self) -> u8 { + self.header.common_header.class_code + } + + fn revision(&self) -> u8 { + self.header.common_header.revision_id + } + + fn irq_type(&self) -> &RwLock { + &self.header.irq_type + } + + fn irq_line(&self) -> u8 { + self.header.interrupt_line + } + + fn interface_code(&self) -> u8 { + self.header.common_header.prog_if + } + + fn subclass(&self) -> u8 { + self.header.common_header.subclass + } } impl Device for PciGeneralDevice { diff --git a/kernel/src/driver/pci/test/pt_device.rs b/kernel/src/driver/pci/test/pt_device.rs index 8c39d5985..78ee71e9b 100644 --- a/kernel/src/driver/pci/test/pt_device.rs +++ b/kernel/src/driver/pci/test/pt_device.rs @@ -14,7 +14,7 @@ use crate::{ kobject::{KObjType, KObject, KObjectCommonData, KObjectState, LockedKObjectState}, kset::KSet, }, - pci::{dev_id::PciDeviceID, device::PciDevice}, + pci::{dev_id::PciDeviceID, device::PciDevice, pci_irq::IrqType}, }, filesystem::{ kernfs::KernFSInode, @@ -36,6 +36,7 @@ pub struct TestDevice { device_data: RwLock, kobj_data: RwLock, kobj_state: LockedKObjectState, + static_type: RwLock, } impl TestDevice { @@ -46,6 +47,7 @@ impl TestDevice { device_data: common_dev, kobj_data: common_kobj, kobj_state: LockedKObjectState::new(None), + static_type: RwLock::new(IrqType::Unused), } } } @@ -70,6 +72,30 @@ impl PciDevice for TestDevice { fn subsystem_device(&self) -> u16 { return 0xffff; } + + fn class_code(&self) -> u8 { + return 0xff; + } + + fn irq_line(&self) -> u8 { + return 0xff; + } + + fn revision(&self) -> u8 { + return 0xff; + } + + fn irq_type(&self) -> &RwLock { + return &self.static_type; + } + + fn interface_code(&self) -> u8 { + return 0xff; + } + + fn subclass(&self) -> u8 { + return 0xff; + } } impl Device for TestDevice { diff --git a/kernel/src/driver/virtio/transport.rs b/kernel/src/driver/virtio/transport.rs index 9dab04795..c437024f7 100644 --- a/kernel/src/driver/virtio/transport.rs +++ b/kernel/src/driver/virtio/transport.rs @@ -33,8 +33,7 @@ impl VirtIOTransport { /// 设置中断 pub fn setup_irq(&self, dev_id: Arc) -> Result<(), PciError> { if let VirtIOTransport::Pci(transport) = self { - let mut pci_device_guard = transport.pci_device(); - let standard_device = pci_device_guard.as_standard_device_mut().unwrap(); + let standard_device = transport.pci_device().as_standard_device().unwrap(); standard_device .irq_init(IRQ::PCI_IRQ_MSIX | IRQ::PCI_IRQ_MSI) .ok_or(PciError::PciIrqError(PciIrqError::IrqNotInited))?; diff --git a/kernel/src/driver/virtio/transport_pci.rs b/kernel/src/driver/virtio/transport_pci.rs index 5ecdfed26..26d4dd742 100644 --- a/kernel/src/driver/virtio/transport_pci.rs +++ b/kernel/src/driver/virtio/transport_pci.rs @@ -10,7 +10,6 @@ use crate::driver::pci::root::pci_root_0; use crate::exception::IrqNumber; -use crate::libs::spinlock::{SpinLock, SpinLockGuard}; use crate::libs::volatile::{ volread, volwrite, ReadOnly, Volatile, VolatileReadable, VolatileWritable, WriteOnly, }; @@ -102,7 +101,7 @@ pub struct PciTransport { config_space: Option>, irq: IrqNumber, dev_id: Arc, - device: Arc>, + device: Arc, } impl PciTransport { @@ -117,7 +116,7 @@ impl PciTransport { /// - `irq_number_offset` - Currently, this parameter is just simple make a offset to the irq number, cause it's not be allowed to have the same irq number within different device #[allow(clippy::extra_unused_type_parameters)] pub fn new( - device: &mut PciDeviceStructureGeneralDevice, + device: Arc, dev_id: Arc, ) -> Result { let irq = VIRTIO_RECV_VECTOR; @@ -135,11 +134,12 @@ impl PciTransport { let mut device_cfg = None; device.bar_ioremap().unwrap()?; device.enable_master(); - let standard_device = device.as_standard_device_mut().unwrap(); + let standard_device = device.as_standard_device().unwrap(); // 目前缺少对PCI设备中断号的统一管理，所以这里需要指定一个中断号。不能与其他中断重复 let irq_vector = standard_device.irq_vector_mut().unwrap(); - irq_vector.push(irq); + irq_vector.write().push(irq); + // panic!(); //device_capability为迭代器，遍历其相当于遍历所有的cap空间 for capability in device.capabilities().unwrap() { if capability.id != PCI_CAP_ID_VNDR { @@ -187,7 +187,7 @@ impl PciTransport { } let common_cfg = get_bar_region::<_>( - &device.standard_device_bar, + &device.standard_device_bar.read(), &common_cfg.ok_or(VirtioPciError::MissingCommonConfig)?, )?; @@ -198,14 +198,15 @@ impl PciTransport { )); } //debug!("notify.offset={},notify.length={}",notify_cfg.offset,notify_cfg.length); - let notify_region = get_bar_region_slice::<_>(&device.standard_device_bar, ¬ify_cfg)?; + let notify_region = + get_bar_region_slice::<_>(&device.standard_device_bar.read(), ¬ify_cfg)?; let isr_status = get_bar_region::<_>( - &device.standard_device_bar, + &device.standard_device_bar.read(), &isr_cfg.ok_or(VirtioPciError::MissingIsrConfig)?, )?; let config_space = if let Some(device_cfg) = device_cfg { Some(get_bar_region_slice::<_>( - &device.standard_device_bar, + &device.standard_device_bar.read(), &device_cfg, )?) } else { @@ -221,12 +222,12 @@ impl PciTransport { config_space, irq, dev_id, - device: Arc::new(SpinLock::new(device.clone())), + device, }) } - pub fn pci_device(&self) -> SpinLockGuard { - self.device.lock() + pub fn pci_device(&self) -> Arc { + self.device.clone() } pub fn irq(&self) -> IrqNumber { diff --git a/kernel/src/driver/virtio/virtio.rs b/kernel/src/driver/virtio/virtio.rs index c21592e69..4864f01bb 100644 --- a/kernel/src/driver/virtio/virtio.rs +++ b/kernel/src/driver/virtio/virtio.rs @@ -6,17 +6,15 @@ use crate::driver::base::device::{Device, DeviceId}; use crate::driver::block::virtio_blk::virtio_blk; use crate::driver::net::virtio_net::virtio_net; use crate::driver::pci::pci::{ - get_pci_device_structures_mut_by_vendor_id, PciDeviceStructure, - PciDeviceStructureGeneralDevice, PCI_DEVICE_LINKEDLIST, + get_pci_device_structures_mut_by_vendor_id, PciDeviceStructureGeneralDevice, + PCI_DEVICE_LINKEDLIST, }; use crate::driver::pci::subsys::pci_bus; use crate::driver::virtio::transport::VirtIOTransport; -use crate::libs::rwlock::RwLockWriteGuard; use alloc::string::String; use alloc::sync::Arc; use alloc::vec::Vec; -use alloc::{boxed::Box, collections::LinkedList}; use log::{debug, error, warn}; use virtio_drivers::transport::{DeviceType, Transport}; @@ -29,12 +27,11 @@ pub fn virtio_probe() { #[allow(dead_code)] fn virtio_probe_pci() { - let mut list = PCI_DEVICE_LINKEDLIST.write(); - let virtio_list = virtio_device_search(&mut list); + let virtio_list = virtio_device_search(); for virtio_device in virtio_list { let dev_id = virtio_device.common_header.device_id; let dev_id = DeviceId::new(None, Some(format!("{dev_id}"))).unwrap(); - match PciTransport::new::(virtio_device, dev_id.clone()) { + match PciTransport::new::(virtio_device.clone(), dev_id.clone()) { Ok(mut transport) => { debug!( "Detected virtio PCI device with device type {:?}, features {:#018x}", @@ -87,19 +84,17 @@ pub(super) fn virtio_device_init( /// ## 返回值 /// /// 返回一个包含所有找到的virtio设备的数组 -fn virtio_device_search<'a>( - list: &'a mut RwLockWriteGuard<'_, LinkedList>>, -) -> Vec<&'a mut PciDeviceStructureGeneralDevice> { +fn virtio_device_search() -> Vec> { + let list = &*PCI_DEVICE_LINKEDLIST; let mut virtio_list = Vec::new(); let result = get_pci_device_structures_mut_by_vendor_id(list, 0x1AF4); for device in result { - let standard_device = device.as_standard_device_mut().unwrap(); + let standard_device = device.as_standard_device().unwrap(); let header = &standard_device.common_header; if header.device_id >= 0x1000 && header.device_id <= 0x103F { virtio_list.push(standard_device); } } - return virtio_list; } From 7c28051e8c601312d3d0fd7bcb71bc71450d10c0 Mon Sep 17 00:00:00 2001 From: linfeng Date: Mon, 11 Nov 2024 21:29:15 +0800 Subject: [PATCH 26/30] chore: update toolchain version to 2024-11-05 (#1031) * chore: update toolchain version to 2024-11-05 * update dragon reach to e945c217b3 * update dog to 6f2c0c8f12 --------- Co-authored-by: longjin --- .github/workflows/makefile.yml | 16 +- build-scripts/Makefile | 2 +- kernel/Makefile | 8 +- kernel/crates/bitmap/src/alloc_bitmap.rs | 2 +- kernel/crates/bitmap/src/static_bitmap.rs | 12 +- kernel/crates/crc/src/lib.rs | 1 - kernel/crates/ida/src/lib.rs | 7 +- kernel/crates/intertrait/macros/src/lib.rs | 1 - kernel/crates/intertrait/src/lib.rs | 1 + kernel/crates/klog_types/src/lib.rs | 1 - kernel/crates/rbpf/src/insn_builder.rs | 48 +++--- kernel/crates/rbpf/tests/ubpf_vm.rs | 4 + kernel/crates/rust-slabmalloc/src/lib.rs | 6 +- kernel/crates/rust-slabmalloc/src/pages.rs | 8 +- kernel/crates/rust-slabmalloc/src/sc.rs | 3 + kernel/crates/system_error/Cargo.toml | 2 - kernel/crates/system_error/src/lib.rs | 2 +- kernel/rust-toolchain.toml | 2 +- kernel/src/Makefile | 2 +- kernel/src/arch/riscv64/interrupt/entry.rs | 23 +-- kernel/src/arch/riscv64/process/kthread.rs | 5 +- kernel/src/arch/riscv64/process/mod.rs | 14 +- kernel/src/arch/x86_64/interrupt/entry.rs | 5 +- kernel/src/arch/x86_64/mm/mod.rs | 2 +- kernel/src/arch/x86_64/process/kthread.rs | 5 +- kernel/src/arch/x86_64/process/mod.rs | 5 +- kernel/src/arch/x86_64/smp/mod.rs | 6 +- kernel/src/driver/base/block/block_device.rs | 4 +- kernel/src/driver/base/block/gendisk.rs | 1 + kernel/src/driver/base/block/manager.rs | 1 + kernel/src/driver/base/char/mod.rs | 2 +- kernel/src/driver/block/virtio_blk.rs | 1 + kernel/src/driver/disk/ahci/ahci_inode.rs | 153 ------------------ kernel/src/driver/disk/ahci/hba.rs | 3 +- kernel/src/driver/disk/ahci/mod.rs | 1 - kernel/src/driver/firmware/efi/memmap.rs | 2 +- kernel/src/driver/firmware/efi/tables.rs | 3 +- kernel/src/driver/input/serio/serio_driver.rs | 3 +- kernel/src/driver/net/dma.rs | 8 +- kernel/src/driver/net/loopback.rs | 10 +- kernel/src/driver/net/virtio_net.rs | 10 +- kernel/src/driver/open_firmware/fdt.rs | 3 +- kernel/src/driver/pci/pci.rs | 2 +- kernel/src/driver/pci/root.rs | 2 +- kernel/src/driver/scsi/mod.rs | 1 + kernel/src/driver/video/fbdev/base/fbmem.rs | 1 + kernel/src/driver/virtio/irq.rs | 1 - kernel/src/driver/virtio/virtio_impl.rs | 8 +- kernel/src/exception/irqdata.rs | 2 + kernel/src/exception/irqdesc.rs | 10 ++ kernel/src/exception/irqdomain.rs | 4 + kernel/src/exception/manage.rs | 1 + kernel/src/exception/softirq.rs | 4 +- kernel/src/filesystem/fat/bpb.rs | 8 +- kernel/src/filesystem/fat/entry.rs | 8 +- kernel/src/filesystem/fat/fs.rs | 18 ++- kernel/src/filesystem/mbr.rs | 2 +- kernel/src/filesystem/overlayfs/mod.rs | 3 +- kernel/src/filesystem/procfs/syscall.rs | 1 - kernel/src/filesystem/vfs/file.rs | 2 +- kernel/src/init/boot.rs | 1 + kernel/src/init/cmdline.rs | 3 + kernel/src/ipc/signal_types.rs | 1 - kernel/src/lib.rs | 22 ++- kernel/src/libs/casting.rs | 1 - kernel/src/libs/cpumask.rs | 2 +- kernel/src/libs/futex/futex.rs | 2 +- kernel/src/libs/lib_ui/screen_manager.rs | 2 - kernel/src/libs/lib_ui/textui.rs | 6 - kernel/src/libs/rbtree.rs | 16 +- kernel/src/libs/rwlock.rs | 15 +- kernel/src/libs/volatile.rs | 1 - kernel/src/mm/memblock.rs | 4 +- kernel/src/perf/bpf.rs | 2 + kernel/src/perf/util.rs | 2 +- kernel/src/process/exit.rs | 2 +- kernel/src/process/syscall.rs | 2 +- kernel/src/syscall/misc.rs | 3 +- kernel/src/syscall/user_access.rs | 4 +- tools/BUILD_CONTAINER_VERSION | 2 +- tools/Makefile | 2 +- tools/bootstrap.sh | 16 +- tools/debugging/logmonitor/src/app.rs | 2 +- user/apps/clear/Makefile | 2 +- user/apps/test-backlog/Makefile | 2 +- user/apps/test-blockcache/Makefile | 2 +- user/apps/test-for-robustfutex/Makefile | 2 +- user/apps/test-mount/Makefile | 2 +- user/apps/test_alarm/Makefile | 2 +- user/apps/test_ebpf/Makefile | 2 +- .../syscall_ebpf-ebpf/rust-toolchain.toml | 2 +- user/apps/test_namespace/Makefile | 2 +- user/apps/test_namespace/makefile.toml | 2 +- user/apps/test_socket/Makefile | 2 +- user/apps/test_statx/Makefile | 2 +- user/apps/test_tokio/Makefile | 2 +- user/apps/user-manage/Makefile | 2 +- user/dadk/config/dog_0_1_0.dadk | 2 +- user/dadk/config/dragon_reach-0.1.0.dadk | 2 +- 99 files changed, 242 insertions(+), 379 deletions(-) delete mode 100644 kernel/src/driver/disk/ahci/ahci_inode.rs diff --git a/.github/workflows/makefile.yml b/.github/workflows/makefile.yml index 6a1612e4a..ecd368a98 100644 --- a/.github/workflows/makefile.yml +++ b/.github/workflows/makefile.yml @@ -12,14 +12,14 @@ jobs: name: Format check ${{ matrix.arch }} runs-on: ubuntu-latest continue-on-error: true - container: dragonos/dragonos-dev:v1.5 + container: dragonos/dragonos-dev:v1.6 strategy: matrix: arch: [x86_64, riscv64] steps: - - run: echo "Running in dragonos/dragonos-dev:v1.5" + - run: echo "Running in dragonos/dragonos-dev:v1.6" - uses: actions/checkout@v3 - name: Format check @@ -35,14 +35,14 @@ jobs: name: Kernel static test ${{ matrix.arch }} runs-on: ubuntu-latest continue-on-error: true - container: dragonos/dragonos-dev:v1.5 + container: dragonos/dragonos-dev:v1.6 strategy: matrix: arch: [x86_64, riscv64] steps: - - run: echo "Running in dragonos/dragonos-dev:v1.5" + - run: echo "Running in dragonos/dragonos-dev:v1.6" - uses: actions/checkout@v3 @@ -56,10 +56,10 @@ jobs: build-x86_64: runs-on: ubuntu-latest - container: dragonos/dragonos-dev:v1.5 + container: dragonos/dragonos-dev:v1.6 steps: - - run: echo "Running in dragonos/dragonos-dev:v1.5" + - run: echo "Running in dragonos/dragonos-dev:v1.6" - uses: actions/checkout@v3 - name: build the DragonOS @@ -78,10 +78,10 @@ jobs: build-riscv64: runs-on: ubuntu-latest - container: dragonos/dragonos-dev:v1.5 + container: dragonos/dragonos-dev:v1.6 steps: - - run: echo "Running in dragonos/dragonos-dev:v1.5" + - run: echo "Running in dragonos/dragonos-dev:v1.6" - uses: actions/checkout@v3 with: diff --git a/build-scripts/Makefile b/build-scripts/Makefile index 14c1bd930..35c6edd58 100644 --- a/build-scripts/Makefile +++ b/build-scripts/Makefile @@ -5,4 +5,4 @@ fmt: clean: @cargo clean check: - @cargo +nightly-2024-07-23 check --workspace $(CARGO_ZBUILD) --message-format=json + @cargo +nightly-2024-11-05 check --workspace $(CARGO_ZBUILD) --message-format=json diff --git a/kernel/Makefile b/kernel/Makefile index 90543aafe..ac09fc83c 100644 --- a/kernel/Makefile +++ b/kernel/Makefile @@ -36,14 +36,14 @@ check: ECHO # @echo "Checking kernel... ARCH=$(ARCH)" # @exit 1 ifeq ($(ARCH), x86_64) - RUSTFLAGS="$(RUSTFLAGS)" cargo +nightly-2024-07-23 check --workspace $(CARGO_ZBUILD) --message-format=json --target ./src/$(TARGET_JSON) + RUSTFLAGS="$(RUSTFLAGS)" cargo +nightly-2024-11-05 check --workspace $(CARGO_ZBUILD) --message-format=json --target ./src/$(TARGET_JSON) else ifeq ($(ARCH), riscv64) - RUSTFLAGS="$(RUSTFLAGS)" cargo +nightly-2024-07-23 check --workspace $(CARGO_ZBUILD) --message-format=json --target $(TARGET_JSON) + RUSTFLAGS="$(RUSTFLAGS)" cargo +nightly-2024-11-05 check --workspace $(CARGO_ZBUILD) --message-format=json --target $(TARGET_JSON) endif test: # 测试内核库 - RUSTFLAGS="$(RUSTFLAGS)" cargo +nightly-2024-07-23 test --workspace --exclude dragonos_kernel rbpf + RUSTFLAGS="$(RUSTFLAGS)" cargo +nightly-2024-11-05 test --workspace --exclude dragonos_kernel rbpf test-rbpf: - cd crates/rbpf && RUSTFLAGS="$(RUSTFLAGS)" cargo +nightly-2024-07-23 test --features=std,user,cranelift \ No newline at end of file + cd crates/rbpf && RUSTFLAGS="$(RUSTFLAGS)" cargo +nightly-2024-11-05 test --features=std,user,cranelift \ No newline at end of file diff --git a/kernel/crates/bitmap/src/alloc_bitmap.rs b/kernel/crates/bitmap/src/alloc_bitmap.rs index d78ceb818..4e3238660 100644 --- a/kernel/crates/bitmap/src/alloc_bitmap.rs +++ b/kernel/crates/bitmap/src/alloc_bitmap.rs @@ -13,7 +13,7 @@ pub struct AllocBitmap { impl AllocBitmap { pub fn new(elements: usize) -> Self { - let data = vec![0usize; (elements + usize::BITS as usize - 1) / (usize::BITS as usize)]; + let data = vec![0usize; elements.div_ceil(usize::BITS as usize)]; Self { elements, data, diff --git a/kernel/crates/bitmap/src/static_bitmap.rs b/kernel/crates/bitmap/src/static_bitmap.rs index c391a7da0..9f6fdacb0 100644 --- a/kernel/crates/bitmap/src/static_bitmap.rs +++ b/kernel/crates/bitmap/src/static_bitmap.rs @@ -8,15 +8,15 @@ use crate::{bitmap_core::BitMapCore, traits::BitMapOps}; #[derive(Debug, Clone)] pub struct StaticBitmap where - [(); (N + usize::BITS as usize - 1) / (usize::BITS as usize)]:, + [(); N.div_ceil(usize::BITS as usize)]:, { - pub data: [usize; (N + usize::BITS as usize - 1) / (usize::BITS as usize)], + pub data: [usize; N.div_ceil(usize::BITS as usize)], core: BitMapCore, } impl Default for StaticBitmap where - [(); (N + usize::BITS as usize - 1) / (usize::BITS as usize)]:, + [(); N.div_ceil(usize::BITS as usize)]:, { fn default() -> Self { Self::new() @@ -25,12 +25,12 @@ where impl StaticBitmap where - [(); (N + usize::BITS as usize - 1) / (usize::BITS as usize)]:, + [(); N.div_ceil(usize::BITS as usize)]:, { /// 创建一个新的静态位图 pub const fn new() -> Self { Self { - data: [0; (N + usize::BITS as usize - 1) / (usize::BITS as usize)], + data: [0; N.div_ceil(usize::BITS as usize)], core: BitMapCore::new(), } } @@ -38,7 +38,7 @@ where impl BitMapOps for StaticBitmap where - [(); (N + usize::BITS as usize - 1) / (usize::BITS as usize)]:, + [(); N.div_ceil(usize::BITS as usize)]:, { #[inline] fn get(&self, index: usize) -> Option { diff --git a/kernel/crates/crc/src/lib.rs b/kernel/crates/crc/src/lib.rs index 5f9a2b7c7..2a039223a 100644 --- a/kernel/crates/crc/src/lib.rs +++ b/kernel/crates/crc/src/lib.rs @@ -1,6 +1,5 @@ #![cfg_attr(not(test), no_std)] #![feature(const_for)] -#![feature(const_mut_refs)] #![feature(const_trait_impl)] #![allow(clippy::needless_return)] diff --git a/kernel/crates/ida/src/lib.rs b/kernel/crates/ida/src/lib.rs index d9676bc92..29c89e159 100644 --- a/kernel/crates/ida/src/lib.rs +++ b/kernel/crates/ida/src/lib.rs @@ -16,7 +16,7 @@ struct EmptyIdaItemRef<'a> { _marker: PhantomData<&'a EmptyIdaItem>, } -impl<'a> Deref for EmptyIdaItemRef<'a> { +impl Deref for EmptyIdaItemRef<'_> { type Target = EmptyIdaItem; fn deref(&self) -> &Self::Target { @@ -27,7 +27,10 @@ impl<'a> Deref for EmptyIdaItemRef<'a> { struct EmptyIdaItem; unsafe impl kdepends::xarray::ItemEntry for EmptyIdaItem { - type Ref<'a> = EmptyIdaItemRef<'a> where Self: 'a; + type Ref<'a> + = EmptyIdaItemRef<'a> + where + Self: 'a; fn into_raw(self) -> *const () { core::ptr::null() diff --git a/kernel/crates/intertrait/macros/src/lib.rs b/kernel/crates/intertrait/macros/src/lib.rs index e050e6abb..5a4339052 100644 --- a/kernel/crates/intertrait/macros/src/lib.rs +++ b/kernel/crates/intertrait/macros/src/lib.rs @@ -61,7 +61,6 @@ mod item_type; /// #[derive(std::fmt::Debug)] /// struct Data; /// ``` - #[proc_macro_attribute] pub fn cast_to(args: TokenStream, input: TokenStream) -> TokenStream { match parse::(args) { diff --git a/kernel/crates/intertrait/src/lib.rs b/kernel/crates/intertrait/src/lib.rs index 59e5e5d55..75073dfe3 100644 --- a/kernel/crates/intertrait/src/lib.rs +++ b/kernel/crates/intertrait/src/lib.rs @@ -122,6 +122,7 @@ static CASTER_MAP: once_cell::sync::Lazy> = None; #[cfg(target_os = "none")] +#[allow(static_mut_refs)] pub fn caster_map() -> &'static HashMap<(TypeId, TypeId), BoxedCaster, BuildFastHasher> { return unsafe { CASTER_MAP.as_ref().unwrap_or_else(|| { diff --git a/kernel/crates/klog_types/src/lib.rs b/kernel/crates/klog_types/src/lib.rs index 22db10681..91ab47fa4 100644 --- a/kernel/crates/klog_types/src/lib.rs +++ b/kernel/crates/klog_types/src/lib.rs @@ -1,5 +1,4 @@ #![no_std] -#![feature(const_refs_to_cell)] #![feature(const_size_of_val)] #![allow(clippy::needless_return)] diff --git a/kernel/crates/rbpf/src/insn_builder.rs b/kernel/crates/rbpf/src/insn_builder.rs index 2b8b6c0d8..d67c431be 100644 --- a/kernel/crates/rbpf/src/insn_builder.rs +++ b/kernel/crates/rbpf/src/insn_builder.rs @@ -74,7 +74,7 @@ pub trait IntoBytes { } /// General implementation of `IntoBytes` for `Instruction` -impl<'i, I: Instruction> IntoBytes for &'i I { +impl IntoBytes for &'_ I { type Bytes = Vec; /// transform immutable reference of `Instruction` into `Vec` with size of 8 @@ -347,7 +347,7 @@ impl<'i> Move<'i> { } } -impl<'i> Instruction for Move<'i> { +impl Instruction for Move<'_> { fn opt_code_byte(&self) -> u8 { let op_bits = self.op_bits as u8; let src_bit = self.src_bit as u8; @@ -415,7 +415,7 @@ impl<'i> SwapBytes<'i> { } } -impl<'i> Instruction for SwapBytes<'i> { +impl Instruction for SwapBytes<'_> { fn opt_code_byte(&self) -> u8 { self.endian as u8 } @@ -456,20 +456,20 @@ impl<'i> Load<'i> { } } -impl<'i> Instruction for Load<'i> { +impl Instruction for Load<'_> { fn opt_code_byte(&self) -> u8 { let size = self.mem_size as u8; let addressing = self.addressing as u8; addressing | size | self.source } - fn get_insn_mut(&mut self) -> &mut Insn { - &mut self.insn - } - fn get_insn(&self) -> &Insn { &self.insn } + + fn get_insn_mut(&mut self) -> &mut Insn { + &mut self.insn + } } /// struct representation of STORE instructions @@ -489,19 +489,19 @@ impl<'i> Store<'i> { } } -impl<'i> Instruction for Store<'i> { +impl Instruction for Store<'_> { fn opt_code_byte(&self) -> u8 { let size = self.mem_size as u8; BPF_MEM | BPF_ST | size | self.source } - fn get_insn_mut(&mut self) -> &mut Insn { - &mut self.insn - } - fn get_insn(&self) -> &Insn { &self.insn } + + fn get_insn_mut(&mut self) -> &mut Insn { + &mut self.insn + } } #[derive(Copy, Clone)] @@ -542,20 +542,20 @@ impl<'i> Jump<'i> { } } -impl<'i> Instruction for Jump<'i> { +impl Instruction for Jump<'_> { fn opt_code_byte(&self) -> u8 { let cmp: u8 = self.cond as u8; let src_bit = self.src_bit as u8; cmp | src_bit | BPF_JMP } - fn get_insn_mut(&mut self) -> &mut Insn { - &mut self.insn - } - fn get_insn(&self) -> &Insn { &self.insn } + + fn get_insn_mut(&mut self) -> &mut Insn { + &mut self.insn + } } #[derive(Copy, Clone, PartialEq, Eq)] @@ -602,18 +602,18 @@ impl<'i> FunctionCall<'i> { } } -impl<'i> Instruction for FunctionCall<'i> { +impl Instruction for FunctionCall<'_> { fn opt_code_byte(&self) -> u8 { BPF_CALL | BPF_JMP } - fn get_insn_mut(&mut self) -> &mut Insn { - &mut self.insn - } - fn get_insn(&self) -> &Insn { &self.insn } + + fn get_insn_mut(&mut self) -> &mut Insn { + &mut self.insn + } } /// struct representation of EXIT instruction @@ -631,7 +631,7 @@ impl<'i> Exit<'i> { } } -impl<'i> Instruction for Exit<'i> { +impl Instruction for Exit<'_> { fn opt_code_byte(&self) -> u8 { BPF_EXIT | BPF_JMP } diff --git a/kernel/crates/rbpf/tests/ubpf_vm.rs b/kernel/crates/rbpf/tests/ubpf_vm.rs index 05d8a9baa..4b2f1c36d 100644 --- a/kernel/crates/rbpf/tests/ubpf_vm.rs +++ b/kernel/crates/rbpf/tests/ubpf_vm.rs @@ -2316,8 +2316,12 @@ fn test_vm_stdw() { // If this case is not handled properly in check_mem(), then we may overflow when adding the // context address and the offset, and make the thread panic with "attempt to add with overflow". // Check that we panic with the expected out-of-bounds error. +// +// The new toolchain introduced `assert_unsafe_precondition` which panics with a different message and can't be +// caught by `#[should_panic]`. This is why we use `#[ignore]` here. #[test] #[should_panic(expected = "Error: out of bounds memory store (insn #1)")] +#[ignore] fn test_vm_stdw_add_overflow() { let prog = assemble( " diff --git a/kernel/crates/rust-slabmalloc/src/lib.rs b/kernel/crates/rust-slabmalloc/src/lib.rs index 5b995fd91..a286f311e 100644 --- a/kernel/crates/rust-slabmalloc/src/lib.rs +++ b/kernel/crates/rust-slabmalloc/src/lib.rs @@ -18,11 +18,9 @@ //! # Implementing GlobalAlloc //! See the [global alloc](https://github.com/gz/rust-slabmalloc/tree/master/examples/global_alloc.rs) example. #![allow(unused_features)] -#![cfg_attr(feature = "unstable", feature(const_mut_refs))] #![no_std] #![crate_name = "slabmalloc"] #![crate_type = "lib"] -#![feature(new_uninit)] #![feature(maybe_uninit_as_bytes)] extern crate alloc; @@ -65,6 +63,8 @@ pub enum AllocationError { /// Needs to adhere to safety requirements of a rust allocator (see GlobalAlloc et. al.). pub unsafe trait Allocator<'a> { fn allocate(&mut self, layout: Layout) -> Result, AllocationError>; + /// # Safety + /// The caller must ensure that the memory is valid and that the layout is correct. unsafe fn deallocate( &mut self, ptr: NonNull, @@ -85,5 +85,7 @@ pub unsafe trait Allocator<'a> { /// 将slab_page归还Buddy的回调函数 pub trait CallBack: Send + Sync { + /// # Safety + /// The caller must ensure that the memory is valid and that the size is correct. unsafe fn free_slab_page(&self, _: *mut u8, _: usize) {} } diff --git a/kernel/crates/rust-slabmalloc/src/pages.rs b/kernel/crates/rust-slabmalloc/src/pages.rs index 1e92eb7e0..22f3231f0 100644 --- a/kernel/crates/rust-slabmalloc/src/pages.rs +++ b/kernel/crates/rust-slabmalloc/src/pages.rs @@ -303,10 +303,10 @@ impl<'a> ObjectPage<'a> { } // These needs some more work to be really safe... -unsafe impl<'a> Send for ObjectPage<'a> {} -unsafe impl<'a> Sync for ObjectPage<'a> {} +unsafe impl Send for ObjectPage<'_> {} +unsafe impl Sync for ObjectPage<'_> {} -impl<'a> AllocablePage for ObjectPage<'a> { +impl AllocablePage for ObjectPage<'_> { const SIZE: usize = OBJECT_PAGE_SIZE; fn bitfield(&self) -> &[AtomicU64; 8] { @@ -331,7 +331,7 @@ impl<'a> Default for ObjectPage<'a> { } } -impl<'a> fmt::Debug for ObjectPage<'a> { +impl fmt::Debug for ObjectPage<'_> { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { write!(f, "ObjectPage") } diff --git a/kernel/crates/rust-slabmalloc/src/sc.rs b/kernel/crates/rust-slabmalloc/src/sc.rs index a17f5770b..2c711bb18 100644 --- a/kernel/crates/rust-slabmalloc/src/sc.rs +++ b/kernel/crates/rust-slabmalloc/src/sc.rs @@ -314,6 +314,9 @@ impl<'a, P: AllocablePage> SCAllocator<'a, P> { /// May return an error in case an invalid `layout` is provided. /// The function may also move internal slab pages between lists partial -> empty /// or full -> partial lists. + /// + /// # Safety + /// The caller must ensure that the `layout` is valid. pub unsafe fn deallocate( &mut self, ptr: NonNull, diff --git a/kernel/crates/system_error/Cargo.toml b/kernel/crates/system_error/Cargo.toml index ccca15c6b..d166286a9 100644 --- a/kernel/crates/system_error/Cargo.toml +++ b/kernel/crates/system_error/Cargo.toml @@ -6,7 +6,5 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -kdepends = { path = "../kdepends" } num-traits = { git = "https://git.mirrors.dragonos.org.cn/DragonOS-Community/num-traits.git", rev="1597c1c", default-features = false } -num = { version = "0.4.0", default-features = false } num-derive = "0.3" \ No newline at end of file diff --git a/kernel/crates/system_error/src/lib.rs b/kernel/crates/system_error/src/lib.rs index 3d81cdf80..441b4b13e 100644 --- a/kernel/crates/system_error/src/lib.rs +++ b/kernel/crates/system_error/src/lib.rs @@ -1,7 +1,7 @@ #![no_std] #![allow(clippy::needless_return)] #![allow(clippy::upper_case_acronyms)] - +#![allow(non_local_definitions)] use num_derive::{FromPrimitive, ToPrimitive}; #[repr(i32)] diff --git a/kernel/rust-toolchain.toml b/kernel/rust-toolchain.toml index 325731828..90975d401 100644 --- a/kernel/rust-toolchain.toml +++ b/kernel/rust-toolchain.toml @@ -1,3 +1,3 @@ [toolchain] -channel = "nightly-2024-07-23" +channel = "nightly-2024-11-05" components = ["rust-src", "clippy"] \ No newline at end of file diff --git a/kernel/src/Makefile b/kernel/src/Makefile index 7476a48bf..f78d7cfbb 100644 --- a/kernel/src/Makefile +++ b/kernel/src/Makefile @@ -40,7 +40,7 @@ kernel_subdirs := debug kernel_rust: - RUSTFLAGS="$(RUSTFLAGS)" cargo +nightly-2024-07-23 $(CARGO_ZBUILD) build --release --target $(TARGET_JSON) + RUSTFLAGS="$(RUSTFLAGS)" cargo +nightly-2024-11-05 $(CARGO_ZBUILD) build --release --target $(TARGET_JSON) all: kernel diff --git a/kernel/src/arch/riscv64/interrupt/entry.rs b/kernel/src/arch/riscv64/interrupt/entry.rs index 31dd785bc..bcb77e493 100644 --- a/kernel/src/arch/riscv64/interrupt/entry.rs +++ b/kernel/src/arch/riscv64/interrupt/entry.rs @@ -4,7 +4,6 @@ use crate::arch::{ interrupt::TrapFrame, }; use asm_macros::{restore_from_x6_to_x31, save_from_x6_to_x31}; -use core::arch::asm; use kdepends::memoffset::offset_of; /// Riscv64中断处理入口 @@ -12,7 +11,7 @@ use kdepends::memoffset::offset_of; #[no_mangle] #[repr(align(4))] pub unsafe extern "C" fn handle_exception() -> ! { - asm!( + core::arch::naked_asm!( concat!(" /* * If coming from userspace, preserve the user thread pointer and load @@ -27,15 +26,14 @@ pub unsafe extern "C" fn handle_exception() -> ! { j {_restore_kernel_tpsp} "), csr_scratch = const CSR_SSCRATCH, - _restore_kernel_tpsp = sym _restore_kernel_tpsp, - options(noreturn), + _restore_kernel_tpsp = sym _restore_kernel_tpsp ) } #[naked] #[no_mangle] unsafe extern "C" fn _restore_kernel_tpsp() -> ! { - asm!( + core::arch::naked_asm!( concat!(" // 这次是从内核态进入中断 // 从sscratch寄存器加载当前cpu的上下文 @@ -48,16 +46,14 @@ unsafe extern "C" fn _restore_kernel_tpsp() -> ! { "), csr_scratch = const CSR_SSCRATCH, lc_off_kernel_sp = const offset_of!(LocalContext, kernel_sp), - _save_context = sym _save_context, - - options(noreturn), + _save_context = sym _save_context ) } #[naked] #[no_mangle] unsafe extern "C" fn _save_context() -> ! { - asm!( + core::arch::naked_asm!( concat!(" @@ -164,15 +160,14 @@ unsafe extern "C" fn _save_context() -> ! { csr_epc = const CSR_SEPC, csr_tval = const CSR_STVAL, csr_cause = const CSR_SCAUSE, - csr_scratch = const CSR_SSCRATCH, - options(noreturn), + csr_scratch = const CSR_SSCRATCH ) } #[naked] #[no_mangle] pub unsafe extern "C" fn ret_from_exception() -> ! { - asm!( + core::arch::naked_asm!( concat!(" ld s0, {off_status}(sp) andi s0, s0, {sr_spp} @@ -249,8 +244,6 @@ pub unsafe extern "C" fn ret_from_exception() -> ! { off_t6 = const offset_of!(TrapFrame, t6), off_sp = const offset_of!(TrapFrame, sp), off_tp = const offset_of!(TrapFrame, tp), - off_epc = const offset_of!(TrapFrame, epc), - - options(noreturn), + off_epc = const offset_of!(TrapFrame, epc) ) } diff --git a/kernel/src/arch/riscv64/process/kthread.rs b/kernel/src/arch/riscv64/process/kthread.rs index 610c8acd4..55d2789cb 100644 --- a/kernel/src/arch/riscv64/process/kthread.rs +++ b/kernel/src/arch/riscv64/process/kthread.rs @@ -66,7 +66,7 @@ impl KernelThreadMechanism { pub(super) unsafe extern "C" fn kernel_thread_bootstrap_stage1() { // 这个函数要是naked的，只是因为现在还没有实现，而naked func不能打`unimplemented!()` // 所以先写成了普通函数 - asm!(concat!( + core::arch::naked_asm!(concat!( " ld x3, {off_gp}(sp) ld x5, {off_t0}(sp) @@ -111,8 +111,7 @@ pub(super) unsafe extern "C" fn kernel_thread_bootstrap_stage1() { off_t4 = const offset_of!(TrapFrame, t4), off_t5 = const offset_of!(TrapFrame, t5), off_t6 = const offset_of!(TrapFrame, t6), - stage2_func = sym jump_to_stage2, - options(noreturn), + stage2_func = sym jump_to_stage2 ); } diff --git a/kernel/src/arch/riscv64/process/mod.rs b/kernel/src/arch/riscv64/process/mod.rs index f7d4e2258..61b05fcb6 100644 --- a/kernel/src/arch/riscv64/process/mod.rs +++ b/kernel/src/arch/riscv64/process/mod.rs @@ -78,9 +78,8 @@ pub unsafe fn arch_switch_to_user(trap_frame: TrapFrame) -> ! { #[naked] unsafe extern "C" fn ready_to_switch_to_user(trap_frame: usize, new_pc: usize) -> ! { - asm!( - concat!( - " + core::arch::naked_asm!(concat!( + " // 设置trap frame mv sp, a0 // 设置返回地址 @@ -88,9 +87,7 @@ unsafe extern "C" fn ready_to_switch_to_user(trap_frame: usize, new_pc: usize) - jr a1 " - ), - options(noreturn) - ); + )); } impl ProcessManager { @@ -227,7 +224,7 @@ impl ProcessManager { /// 参考 https://code.dragonos.org.cn/xref/linux-6.6.21/arch/riscv/kernel/entry.S#233 #[naked] unsafe extern "C" fn switch_to_inner(prev: *mut ArchPCBInfo, next: *mut ArchPCBInfo) { - core::arch::asm!(concat!( + core::arch::naked_asm!(concat!( " sd ra, {off_ra}(a0) sd sp, {off_sp}(a0) @@ -304,8 +301,7 @@ unsafe extern "C" fn switch_to_inner(prev: *mut ArchPCBInfo, next: *mut ArchPCBI off_s9 = const(offset_of!(ArchPCBInfo, s9)), off_s10 = const(offset_of!(ArchPCBInfo, s10)), off_s11 = const(offset_of!(ArchPCBInfo, s11)), - before_switch_finish_hook = sym before_switch_finish_hook, - options(noreturn)); + before_switch_finish_hook = sym before_switch_finish_hook); } /// 在切换上下文完成后的钩子函数(必须在这里加一个跳转函数，否则会出现relocation truncated to fit: R_RISCV_JAL错误) diff --git a/kernel/src/arch/x86_64/interrupt/entry.rs b/kernel/src/arch/x86_64/interrupt/entry.rs index 31e53a20f..072ec4ddb 100644 --- a/kernel/src/arch/x86_64/interrupt/entry.rs +++ b/kernel/src/arch/x86_64/interrupt/entry.rs @@ -45,7 +45,7 @@ macro_rules! interrupt_handler { #[naked] #[no_mangle] unsafe extern "C" fn []() { - core::arch::asm!( + core::arch::naked_asm!( concat!( " push 0x0 @@ -60,8 +60,7 @@ macro_rules! interrupt_handler { jmp x86_64_do_irq " ), - irqnum = const($name), - options(noreturn) + irqnum = const($name) ); } } diff --git a/kernel/src/arch/x86_64/mm/mod.rs b/kernel/src/arch/x86_64/mm/mod.rs index f96c96f4d..85753f9ca 100644 --- a/kernel/src/arch/x86_64/mm/mod.rs +++ b/kernel/src/arch/x86_64/mm/mod.rs @@ -500,7 +500,7 @@ unsafe fn allocator_init() { for i in 0..total_num { let area = mem_block_manager().get_initial_memory_region(i).unwrap(); // debug!("area: base={:?}, size={:#x}, end={:?}", area.base, area.size, area.base + area.size); - for i in 0..((area.size + MMArch::PAGE_SIZE - 1) / MMArch::PAGE_SIZE) { + for i in 0..area.size.div_ceil(MMArch::PAGE_SIZE) { let paddr = area.base.add(i * MMArch::PAGE_SIZE); let vaddr = unsafe { MMArch::phys_2_virt(paddr) }.unwrap(); let flags = kernel_page_flags::(vaddr); diff --git a/kernel/src/arch/x86_64/process/kthread.rs b/kernel/src/arch/x86_64/process/kthread.rs index 58f6df1f8..612449e06 100644 --- a/kernel/src/arch/x86_64/process/kthread.rs +++ b/kernel/src/arch/x86_64/process/kthread.rs @@ -1,5 +1,3 @@ -use core::arch::asm; - use alloc::sync::Arc; use system_error::SystemError; @@ -61,7 +59,7 @@ impl KernelThreadMechanism { /// 跳转之后，指向Box的指针将传入到stage2的函数 #[naked] pub(super) unsafe extern "sysv64" fn kernel_thread_bootstrap_stage1() { - asm!( + core::arch::naked_asm!( concat!( " @@ -92,6 +90,5 @@ pub(super) unsafe extern "sysv64" fn kernel_thread_bootstrap_stage1() { " ), stage2_func = sym kernel_thread_bootstrap_stage2, - options(noreturn) ) } diff --git a/kernel/src/arch/x86_64/process/mod.rs b/kernel/src/arch/x86_64/process/mod.rs index be2f53387..674852d07 100644 --- a/kernel/src/arch/x86_64/process/mod.rs +++ b/kernel/src/arch/x86_64/process/mod.rs @@ -425,7 +425,7 @@ impl ProcessManager { /// 保存上下文，然后切换进程，接着jmp到`switch_finish_hook`钩子函数 #[naked] unsafe extern "sysv64" fn switch_to_inner(prev: *mut ArchPCBInfo, next: *mut ArchPCBInfo) { - asm!( + core::arch::naked_asm!( // As a quick reminder for those who are unfamiliar with the System V ABI (extern "C"): // // - the current parameters are passed in the registers `rdi`, `rsi`, @@ -498,13 +498,12 @@ unsafe extern "sysv64" fn switch_to_inner(prev: *mut ArchPCBInfo, next: *mut Arc off_gs = const(offset_of!(ArchPCBInfo, gs)), switch_hook = sym crate::process::switch_finish_hook, - options(noreturn), ); } #[naked] unsafe extern "sysv64" fn switch_back() -> ! { - asm!("ret", options(noreturn)); + core::arch::naked_asm!("ret"); } pub unsafe fn arch_switch_to_user(trap_frame: TrapFrame) -> ! { diff --git a/kernel/src/arch/x86_64/smp/mod.rs b/kernel/src/arch/x86_64/smp/mod.rs index 2eaa5fc74..eec877b86 100644 --- a/kernel/src/arch/x86_64/smp/mod.rs +++ b/kernel/src/arch/x86_64/smp/mod.rs @@ -1,5 +1,4 @@ use core::{ - arch::asm, hint::spin_loop, sync::atomic::{compiler_fence, fence, AtomicBool, Ordering}, }; @@ -65,14 +64,13 @@ unsafe extern "C" fn smp_ap_start() -> ! { #[naked] unsafe extern "sysv64" fn smp_init_switch_stack(st: &ApStartStackInfo) -> ! { - asm!(concat!(" + core::arch::naked_asm!(concat!(" mov rsp, [rdi + {off_rsp}] mov rbp, [rdi + {off_rsp}] jmp {stage1} "), off_rsp = const(offset_of!(ApStartStackInfo, vaddr)), - stage1 = sym smp_ap_start_stage1, - options(noreturn)); + stage1 = sym smp_ap_start_stage1); } unsafe extern "C" fn smp_ap_start_stage1() -> ! { diff --git a/kernel/src/driver/base/block/block_device.rs b/kernel/src/driver/base/block/block_device.rs index 2a9a9b1e9..c02464a0b 100644 --- a/kernel/src/driver/base/block/block_device.rs +++ b/kernel/src/driver/base/block/block_device.rs @@ -20,8 +20,8 @@ use system_error::SystemError; use super::{disk_info::Partition, gendisk::GenDisk, manager::BlockDevMeta}; -/// 该文件定义了 Device 和 BlockDevice 的接口 -/// Notice 设备错误码使用 Posix 规定的 int32_t 的错误码表示，而不是自己定义错误enum +// 该文件定义了 Device 和 BlockDevice 的接口 +// Notice 设备错误码使用 Posix 规定的 int32_t 的错误码表示，而不是自己定义错误enum // 使用方法: // 假设 blk_dev 是块设备 diff --git a/kernel/src/driver/base/block/gendisk.rs b/kernel/src/driver/base/block/gendisk.rs index 699d01af0..ba2818391 100644 --- a/kernel/src/driver/base/block/gendisk.rs +++ b/kernel/src/driver/base/block/gendisk.rs @@ -161,6 +161,7 @@ impl GenDiskMap { } #[inline] + #[allow(dead_code)] pub fn max_idx(&self) -> u32 { self.max_idx.load(Ordering::SeqCst) } diff --git a/kernel/src/driver/base/block/manager.rs b/kernel/src/driver/base/block/manager.rs index 5ebda4c7b..7fe6c2919 100644 --- a/kernel/src/driver/base/block/manager.rs +++ b/kernel/src/driver/base/block/manager.rs @@ -134,6 +134,7 @@ impl BlockDevManager { } /// 卸载磁盘设备 + #[allow(dead_code)] pub fn unregister(&self, dev: &Arc) { let mut inner = self.inner(); inner.disks.remove(dev.dev_name()); diff --git a/kernel/src/driver/base/char/mod.rs b/kernel/src/driver/base/char/mod.rs index 090abef18..f6ca986ba 100644 --- a/kernel/src/driver/base/char/mod.rs +++ b/kernel/src/driver/base/char/mod.rs @@ -18,7 +18,7 @@ use super::{ pub trait CharDevice: Device { /// Notice buffer对应设备按字节划分，使用u8类型 /// Notice offset应该从0开始计数 - + /// /// @brief: 从设备的第offset个字节开始，读取len个byte，存放到buf中 /// @parameter offset: 起始字节偏移量 /// @parameter len: 读取字节的数量 diff --git a/kernel/src/driver/block/virtio_blk.rs b/kernel/src/driver/block/virtio_blk.rs index 6240263b5..ce8ac62c1 100644 --- a/kernel/src/driver/block/virtio_blk.rs +++ b/kernel/src/driver/block/virtio_blk.rs @@ -137,6 +137,7 @@ impl VirtIOBlkManager { BlockDevName::new(format!("vd{}", x), id) } + #[allow(dead_code)] pub fn free_id(&self, id: usize) { if id >= Self::MAX_DEVICES { return; diff --git a/kernel/src/driver/disk/ahci/ahci_inode.rs b/kernel/src/driver/disk/ahci/ahci_inode.rs deleted file mode 100644 index ec0bed271..000000000 --- a/kernel/src/driver/disk/ahci/ahci_inode.rs +++ /dev/null @@ -1,153 +0,0 @@ -use crate::driver::base::block::block_device::BlockDevice; -use crate::driver::base::device::device_number::{DeviceNumber, Major}; -use crate::filesystem::devfs::{DevFS, DeviceINode}; -use crate::filesystem::vfs::file::FileMode; -use crate::filesystem::vfs::syscall::ModeType; -use crate::filesystem::vfs::{ - core::generate_inode_id, FilePrivateData, FileSystem, FileType, IndexNode, Metadata, -}; -use crate::libs::spinlock::SpinLockGuard; -use crate::{libs::spinlock::SpinLock, time::PosixTimeSpec}; -use alloc::{ - string::String, - sync::{Arc, Weak}, - vec::Vec, -}; -use system_error::SystemError; - -use super::ahcidisk::LockedAhciDisk; - -#[derive(Debug)] -pub struct AhciInode { - /// uuid 暂时不知道有什么用（x - // uuid: Uuid, - /// 指向自身的弱引用 - self_ref: Weak, - /// 指向inode所在的文件系统对象的指针 - fs: Weak, - /// INode 元数据 - metadata: Metadata, - /// INode 对应的磁盘 - disk: Arc, -} - -#[derive(Debug)] -pub struct LockedAhciInode(pub SpinLock); - -impl LockedAhciInode { - pub fn new(disk: Arc) -> Arc { - let inode = AhciInode { - // uuid: Uuid::new_v5(), - self_ref: Weak::default(), - fs: Weak::default(), - disk, - metadata: Metadata { - dev_id: 1, - inode_id: generate_inode_id(), - size: 0, - blk_size: 0, - blocks: 0, - atime: PosixTimeSpec::default(), - mtime: PosixTimeSpec::default(), - ctime: PosixTimeSpec::default(), - file_type: FileType::BlockDevice, // 文件夹，block设备，char设备 - mode: ModeType::from_bits_truncate(0o666), - nlinks: 1, - uid: 0, - gid: 0, - raw_dev: DeviceNumber::new(Major::HD_MAJOR, 0), - }, - }; - - let result = Arc::new(LockedAhciInode(SpinLock::new(inode))); - result.0.lock().self_ref = Arc::downgrade(&result); - - return result; - } -} - -impl DeviceINode for LockedAhciInode { - fn set_fs(&self, fs: Weak) { - self.0.lock().fs = fs; - } -} - -impl IndexNode for LockedAhciInode { - fn as_any_ref(&self) -> &dyn core::any::Any { - self - } - - fn open( - &self, - _data: SpinLockGuard, - _mode: &FileMode, - ) -> Result<(), SystemError> { - Err(SystemError::ENOSYS) - } - - fn close(&self, _data: SpinLockGuard) -> Result<(), SystemError> { - Err(SystemError::ENOSYS) - } - - fn metadata(&self) -> Result { - return Ok(self.0.lock().metadata.clone()); - } - - fn fs(&self) -> Arc { - return self.0.lock().fs.upgrade().unwrap(); - } - - fn list(&self) -> Result, SystemError> { - Err(SystemError::ENOSYS) - } - - fn set_metadata(&self, metadata: &Metadata) -> Result<(), SystemError> { - let mut inode = self.0.lock(); - inode.metadata.atime = metadata.atime; - inode.metadata.mtime = metadata.mtime; - inode.metadata.ctime = metadata.ctime; - inode.metadata.mode = metadata.mode; - inode.metadata.uid = metadata.uid; - inode.metadata.gid = metadata.gid; - - return Ok(()); - } - - /// 读设备 - 应该调用设备的函数读写，而不是通过文件系统读写 - fn read_at( - &self, - offset: usize, // lba地址 - len: usize, - buf: &mut [u8], - data: SpinLockGuard, - ) -> Result { - if buf.len() < len { - return Err(SystemError::EINVAL); - } - - if let FilePrivateData::Unused = *data { - return self.0.lock().disk.read_at_bytes(offset, len, buf); - } - - return Err(SystemError::EINVAL); - } - - /// 写设备 - 应该调用设备的函数读写，而不是通过文件系统读写 - fn write_at( - &self, - offset: usize, // lba地址 - len: usize, - buf: &[u8], - data: SpinLockGuard, - ) -> Result { - if buf.len() < len { - return Err(SystemError::EINVAL); - } - - if let FilePrivateData::Unused = *data { - return self.0.lock().disk.write_at_bytes(offset, len, buf); - } - - return Err(SystemError::EINVAL); - } -} diff --git a/kernel/src/driver/disk/ahci/hba.rs b/kernel/src/driver/disk/ahci/hba.rs index aa324e654..64be8c6bf 100644 --- a/kernel/src/driver/disk/ahci/hba.rs +++ b/kernel/src/driver/disk/ahci/hba.rs @@ -1,3 +1,4 @@ +//! 文件说明: 实现了 AHCI 中的控制器 HBA 的相关行为 use core::{intrinsics::size_of, ptr}; use core::sync::atomic::compiler_fence; @@ -5,8 +6,6 @@ use core::sync::atomic::compiler_fence; use crate::arch::MMArch; use crate::mm::{MemoryManagementArch, PhysAddr}; -/// 文件说明: 实现了 AHCI 中的控制器 HBA 的相关行为 - /// 根据 AHCI 写出 HBA 的 Command pub const ATA_CMD_READ_DMA_EXT: u8 = 0x25; // 读操作，并且退出 pub const ATA_CMD_WRITE_DMA_EXT: u8 = 0x35; // 写操作，并且退出 diff --git a/kernel/src/driver/disk/ahci/mod.rs b/kernel/src/driver/disk/ahci/mod.rs index 38e036d60..8af505816 100644 --- a/kernel/src/driver/disk/ahci/mod.rs +++ b/kernel/src/driver/disk/ahci/mod.rs @@ -1,5 +1,4 @@ // 导出 ahci 相关的 module -pub mod ahci_inode; pub mod ahcidisk; pub mod hba; use crate::arch::MMArch; diff --git a/kernel/src/driver/firmware/efi/memmap.rs b/kernel/src/driver/firmware/efi/memmap.rs index c213d853b..afada046a 100644 --- a/kernel/src/driver/firmware/efi/memmap.rs +++ b/kernel/src/driver/firmware/efi/memmap.rs @@ -67,7 +67,7 @@ impl<'a> EFIMemoryDescIter<'a> { } } -impl<'a> Iterator for EFIMemoryDescIter<'a> { +impl Iterator for EFIMemoryDescIter<'_> { type Item = MemoryDescriptor; fn next(&mut self) -> Option { diff --git a/kernel/src/driver/firmware/efi/tables.rs b/kernel/src/driver/firmware/efi/tables.rs index a6e9c3448..770c4a048 100644 --- a/kernel/src/driver/firmware/efi/tables.rs +++ b/kernel/src/driver/firmware/efi/tables.rs @@ -74,8 +74,7 @@ impl EFIManager { warn!("report systable header: failed to map systable header, err: {fw_ptr:?}"); } - let s = CStr::from_bytes_with_nul(&tmp_buf) - .unwrap_or_else(|_| CStr::from_bytes_with_nul(b"Unknown\0").unwrap()); + let s = CStr::from_bytes_with_nul(&tmp_buf).unwrap_or(c"Unknown"); info!("EFI version: {:?}, vendor: {:?}", header.revision, s); } diff --git a/kernel/src/driver/input/serio/serio_driver.rs b/kernel/src/driver/input/serio/serio_driver.rs index c076a5c44..e491b1840 100644 --- a/kernel/src/driver/input/serio/serio_driver.rs +++ b/kernel/src/driver/input/serio/serio_driver.rs @@ -45,8 +45,7 @@ pub trait SerioDriver: Driver { fn cleanup(&self, device: &Arc) -> Result<(), SystemError>; } -///todo: https://code.dragonos.org.cn/xref/linux-6.1.9/drivers/input/serio/serio.c#810 - +/// todo: https://code.dragonos.org.cn/xref/linux-6.1.9/drivers/input/serio/serio.c#810 #[allow(dead_code)] #[inline(always)] pub fn serio_driver_manager() -> &'static SerioDriverManager { diff --git a/kernel/src/driver/net/dma.rs b/kernel/src/driver/net/dma.rs index f8c06b74e..e03fe86a5 100644 --- a/kernel/src/driver/net/dma.rs +++ b/kernel/src/driver/net/dma.rs @@ -17,7 +17,9 @@ const PAGE_SIZE: usize = 4096; /// @return PhysAddr 获得的内存页的初始物理地址 pub fn dma_alloc(pages: usize) -> (usize, NonNull) { let page_num = PageFrameCount::new( - ((pages * PAGE_SIZE + MMArch::PAGE_SIZE - 1) / MMArch::PAGE_SIZE).next_power_of_two(), + (pages * PAGE_SIZE) + .div_ceil(MMArch::PAGE_SIZE) + .next_power_of_two(), ); unsafe { let (paddr, count) = allocate_page_frames(page_num).expect("e1000e: alloc page failed"); @@ -44,7 +46,9 @@ pub fn dma_alloc(pages: usize) -> (usize, NonNull) { /// @return i32 0表示成功 pub unsafe fn dma_dealloc(paddr: usize, vaddr: NonNull, pages: usize) -> i32 { let page_count = PageFrameCount::new( - ((pages * PAGE_SIZE + MMArch::PAGE_SIZE - 1) / MMArch::PAGE_SIZE).next_power_of_two(), + (pages * PAGE_SIZE) + .div_ceil(MMArch::PAGE_SIZE) + .next_power_of_two(), ); // 恢复页面属性 diff --git a/kernel/src/driver/net/loopback.rs b/kernel/src/driver/net/loopback.rs index f8861d43f..cb251ce4a 100644 --- a/kernel/src/driver/net/loopback.rs +++ b/kernel/src/driver/net/loopback.rs @@ -186,8 +186,14 @@ impl Clone for LoopbackDriver { } impl phy::Device for LoopbackDriver { - type RxToken<'a> = LoopbackRxToken where Self: 'a; - type TxToken<'a> = LoopbackTxToken where Self: 'a; + type RxToken<'a> + = LoopbackRxToken + where + Self: 'a; + type TxToken<'a> + = LoopbackTxToken + where + Self: 'a; /// ## 返回设备的物理层特性。 /// lo设备的最大传输单元为65535，最大突发大小为1，传输介质默认为Ethernet fn capabilities(&self) -> phy::DeviceCapabilities { diff --git a/kernel/src/driver/net/virtio_net.rs b/kernel/src/driver/net/virtio_net.rs index 521f2752b..35190e2b9 100644 --- a/kernel/src/driver/net/virtio_net.rs +++ b/kernel/src/driver/net/virtio_net.rs @@ -539,8 +539,14 @@ impl VirtioNetToken { } impl phy::Device for VirtIONicDeviceInner { - type RxToken<'a> = VirtioNetToken where Self: 'a; - type TxToken<'a> = VirtioNetToken where Self: 'a; + type RxToken<'a> + = VirtioNetToken + where + Self: 'a; + type TxToken<'a> + = VirtioNetToken + where + Self: 'a; fn receive( &mut self, diff --git a/kernel/src/driver/open_firmware/fdt.rs b/kernel/src/driver/open_firmware/fdt.rs index aff55017d..deefb05bc 100644 --- a/kernel/src/driver/open_firmware/fdt.rs +++ b/kernel/src/driver/open_firmware/fdt.rs @@ -71,6 +71,7 @@ impl OpenFirmwareFdtDriver { return Ok(()); } + #[allow(dead_code)] pub unsafe fn set_fdt_map_guard(&self, guard: Option) { self.inner.write().fdt_map_guard = guard; } @@ -293,6 +294,7 @@ impl OpenFirmwareFdtDriver { /// 在UEFI初始化后，扫描FDT中的`/reserved-memory`节点，设置保留的内存 /// /// 参考： https://code.dragonos.org.cn/xref/linux-6.1.9/drivers/of/fdt.c#634 + #[allow(dead_code)] pub fn early_init_fdt_scan_reserved_mem(&self) { let vaddr = boot_params().read().fdt(); if vaddr.is_none() { @@ -323,7 +325,6 @@ impl OpenFirmwareFdtDriver { } /// 保留fdt自身的内存空间 - fn early_reserve_fdt_itself(&self, fdt: &Fdt) { #[cfg(target_arch = "riscv64")] { diff --git a/kernel/src/driver/pci/pci.rs b/kernel/src/driver/pci/pci.rs index 0a22fe6da..82bb20c9e 100644 --- a/kernel/src/driver/pci/pci.rs +++ b/kernel/src/driver/pci/pci.rs @@ -1474,7 +1474,7 @@ pub struct ExternalCapabilityIterator<'a> { pub bus_device_function: BusDeviceFunction, pub next_capability_offset: Option, } -impl<'a> Iterator for ExternalCapabilityIterator<'a> { +impl Iterator for ExternalCapabilityIterator<'_> { type Item = ExternalCapabilityInfo; fn next(&mut self) -> Option { let offset = self.next_capability_offset?; diff --git a/kernel/src/driver/pci/root.rs b/kernel/src/driver/pci/root.rs index c74c6cf9b..40c2cbeaa 100644 --- a/kernel/src/driver/pci/root.rs +++ b/kernel/src/driver/pci/root.rs @@ -317,7 +317,7 @@ pub struct PciRootIterator<'a> { index: usize, } -impl<'a> Iterator for PciRootIterator<'a> { +impl Iterator for PciRootIterator<'_> { type Item = Arc; fn next(&mut self) -> Option { diff --git a/kernel/src/driver/scsi/mod.rs b/kernel/src/driver/scsi/mod.rs index 83e6be572..21a42cc85 100644 --- a/kernel/src/driver/scsi/mod.rs +++ b/kernel/src/driver/scsi/mod.rs @@ -64,6 +64,7 @@ impl ScsiManager { BlockDevName::new(format!("sd{}", x), id) } + #[allow(dead_code)] pub fn free_id(&self, id: usize) { if id >= Self::MAX_DEVICES { return; diff --git a/kernel/src/driver/video/fbdev/base/fbmem.rs b/kernel/src/driver/video/fbdev/base/fbmem.rs index bcea81ac8..c3749015d 100644 --- a/kernel/src/driver/video/fbdev/base/fbmem.rs +++ b/kernel/src/driver/video/fbdev/base/fbmem.rs @@ -181,6 +181,7 @@ impl FrameBufferManager { } /// 根据id查找帧缓冲区 + #[allow(dead_code)] pub fn find_fb_by_id(&self, id: FbId) -> Result>, SystemError> { if unlikely(!id.is_valid()) { return Err(SystemError::EINVAL); diff --git a/kernel/src/driver/virtio/irq.rs b/kernel/src/driver/virtio/irq.rs index c5d075eb0..3a4a86da3 100644 --- a/kernel/src/driver/virtio/irq.rs +++ b/kernel/src/driver/virtio/irq.rs @@ -77,7 +77,6 @@ impl VirtIOIrqManager { /// # 返回 /// - 如果找到了设备，返回一个包含设备的`Option>`。 /// - 如果没有找到设备，返回`None`。 - pub fn lookup_device(&self, dev_id: &Arc) -> Option> { let map = self.map.read_irqsave(); map.get(dev_id).cloned() diff --git a/kernel/src/driver/virtio/virtio_impl.rs b/kernel/src/driver/virtio/virtio_impl.rs index 0166b138b..0608a0ce4 100644 --- a/kernel/src/driver/virtio/virtio_impl.rs +++ b/kernel/src/driver/virtio/virtio_impl.rs @@ -23,7 +23,9 @@ unsafe impl Hal for HalImpl { _direction: BufferDirection, ) -> (virtio_drivers::PhysAddr, NonNull) { let page_num = PageFrameCount::new( - ((pages * PAGE_SIZE + MMArch::PAGE_SIZE - 1) / MMArch::PAGE_SIZE).next_power_of_two(), + (pages * PAGE_SIZE) + .div_ceil(MMArch::PAGE_SIZE) + .next_power_of_two(), ); unsafe { let (paddr, count) = @@ -55,7 +57,9 @@ unsafe impl Hal for HalImpl { pages: usize, ) -> i32 { let page_count = PageFrameCount::new( - ((pages * PAGE_SIZE + MMArch::PAGE_SIZE - 1) / MMArch::PAGE_SIZE).next_power_of_two(), + (pages * PAGE_SIZE) + .div_ceil(MMArch::PAGE_SIZE) + .next_power_of_two(), ); // 恢复页面属性 diff --git a/kernel/src/exception/irqdata.rs b/kernel/src/exception/irqdata.rs index 2388907cb..ca8da4cdc 100644 --- a/kernel/src/exception/irqdata.rs +++ b/kernel/src/exception/irqdata.rs @@ -306,6 +306,7 @@ impl IrqCommonData { self.inner.lock_irqsave().affinity = affinity; } + #[allow(dead_code)] pub fn set_effective_affinity(&self, affinity: CpuMask) { self.inner.lock_irqsave().effective_affinity = affinity; } @@ -346,6 +347,7 @@ impl InnerIrqCommonData { self.handler_data.clone() } + #[allow(dead_code)] pub fn effective_affinity(&self) -> &CpuMask { &self.effective_affinity } diff --git a/kernel/src/exception/irqdesc.rs b/kernel/src/exception/irqdesc.rs index 815655965..96ea15797 100644 --- a/kernel/src/exception/irqdesc.rs +++ b/kernel/src/exception/irqdesc.rs @@ -286,10 +286,12 @@ impl IrqDesc { ); } + #[allow(dead_code)] pub fn set_probe(&self) { self.modify_status(IrqLineStatus::IRQ_NOPROBE, IrqLineStatus::empty()); } + #[allow(dead_code)] pub fn set_noprobe(&self) { self.modify_status(IrqLineStatus::empty(), IrqLineStatus::IRQ_NOPROBE); } @@ -416,6 +418,7 @@ impl InnerIrqDesc { self.line_status.insert(IrqLineStatus::IRQ_NOTHREAD); } + #[allow(dead_code)] pub fn clear_nothread(&mut self) { self.line_status.remove(IrqLineStatus::IRQ_NOTHREAD); } @@ -451,6 +454,7 @@ impl InnerIrqDesc { !self.line_status.contains(IrqLineStatus::IRQ_NOAUTOEN) } + #[allow(dead_code)] pub fn can_thread(&self) -> bool { !self.line_status.contains(IrqLineStatus::IRQ_NOTHREAD) } @@ -486,6 +490,7 @@ impl InnerIrqDesc { self.actions.clear(); } + #[allow(dead_code)] pub fn remove_action(&mut self, action: &Arc) { self.actions.retain(|a| !Arc::ptr_eq(a, action)); } @@ -506,14 +511,17 @@ impl InnerIrqDesc { &self.common_data } + #[allow(dead_code)] pub fn depth(&self) -> u32 { self.depth } + #[allow(dead_code)] pub fn wake_depth(&self) -> u32 { self.wake_depth } + #[allow(dead_code)] pub fn set_depth(&mut self, depth: u32) { self.depth = depth; } @@ -540,6 +548,7 @@ impl InnerIrqDesc { &mut self.percpu_enabled } + #[allow(dead_code)] pub fn percpu_affinity(&self) -> &Option { &self.percpu_affinity } @@ -969,6 +978,7 @@ impl IrqDescManager { } /// 设置指定irq的可用cpu为所有cpu + #[allow(dead_code)] pub fn set_percpu_devid_all(&self, irq: IrqNumber) -> Result<(), SystemError> { self.set_percpu_devid(irq, None) } diff --git a/kernel/src/exception/irqdomain.rs b/kernel/src/exception/irqdomain.rs index 021979dd8..814884dd5 100644 --- a/kernel/src/exception/irqdomain.rs +++ b/kernel/src/exception/irqdomain.rs @@ -319,6 +319,7 @@ impl IrqDomainManager { /// - `handler_data`: 中断流处理程序数据 /// - `handler_name`: 中断处理程序名称 #[allow(clippy::too_many_arguments)] + #[allow(dead_code)] pub fn domain_set_info( &self, domain: &Arc, @@ -557,14 +558,17 @@ impl IrqDomain { } /// The number of mapped interrupts + #[allow(dead_code)] pub fn map_count(&self) -> u32 { self.revmap_read_irqsave().map.len() as u32 } + #[allow(dead_code)] pub fn host_data(&self) -> Option> { self.inner.lock_irqsave().host_data.clone() } + #[allow(dead_code)] pub fn set_host_data(&self, host_data: Option>) { self.inner.lock_irqsave().host_data = host_data; } diff --git a/kernel/src/exception/manage.rs b/kernel/src/exception/manage.rs index 851a4024f..4e84d8a68 100644 --- a/kernel/src/exception/manage.rs +++ b/kernel/src/exception/manage.rs @@ -785,6 +785,7 @@ impl IrqManager { ); } + #[allow(dead_code)] pub fn irq_set_affinity( &self, irq_data: &Arc, diff --git a/kernel/src/exception/softirq.rs b/kernel/src/exception/softirq.rs index f7d9513a5..c67cc23b0 100644 --- a/kernel/src/exception/softirq.rs +++ b/kernel/src/exception/softirq.rs @@ -275,13 +275,13 @@ struct RunningCountGuard<'a> { } impl<'a> RunningCountGuard<'a> { - fn new(cpu_running_count: &'a PerCpuVar) -> RunningCountGuard { + fn new(cpu_running_count: &'a PerCpuVar) -> RunningCountGuard<'a> { cpu_running_count.get().fetch_add(1, Ordering::SeqCst); return RunningCountGuard { cpu_running_count }; } } -impl<'a> Drop for RunningCountGuard<'a> { +impl Drop for RunningCountGuard<'_> { fn drop(&mut self) { self.cpu_running_count.get().fetch_sub(1, Ordering::SeqCst); } diff --git a/kernel/src/filesystem/fat/bpb.rs b/kernel/src/filesystem/fat/bpb.rs index c31e3a560..4add07d81 100644 --- a/kernel/src/filesystem/fat/bpb.rs +++ b/kernel/src/filesystem/fat/bpb.rs @@ -266,8 +266,7 @@ impl BiosParameterBlock { bpb.trail_sig = cursor.read_u16()?; // 计算根目录项占用的空间（单位：字节） - let root_sectors = ((bpb.root_entries_cnt as u32 * 32) + (bpb.bytes_per_sector as u32 - 1)) - / (bpb.bytes_per_sector as u32); + let root_sectors = (bpb.root_entries_cnt as u32 * 32).div_ceil(bpb.bytes_per_sector as u32); // 每FAT扇区数 let fat_size = if bpb.fat_size_16 != 0 { @@ -347,9 +346,8 @@ impl BiosParameterBlock { } }; - let root_sectors = ((self.root_entries_cnt as u32 * 32) - + (self.bytes_per_sector as u32 - 1)) - / (self.bytes_per_sector as u32); + let root_sectors = + (self.root_entries_cnt as u32 * 32).div_ceil(self.bytes_per_sector as u32); // 当前分区总扇区数 let total_sectors = if self.total_sectors_16 != 0 { diff --git a/kernel/src/filesystem/fat/entry.rs b/kernel/src/filesystem/fat/entry.rs index 0e825bf05..d6f69e7cb 100644 --- a/kernel/src/filesystem/fat/entry.rs +++ b/kernel/src/filesystem/fat/entry.rs @@ -256,8 +256,7 @@ impl FATFile { // 如果还需要更多的簇 if bytes_remain_in_cluster < extra_bytes { let clusters_to_allocate = - (extra_bytes - bytes_remain_in_cluster + fs.bytes_per_cluster() - 1) - / fs.bytes_per_cluster(); + (extra_bytes - bytes_remain_in_cluster).div_ceil(fs.bytes_per_cluster()); let last_cluster = if let Some(c) = fs.get_last_cluster(self.first_cluster) { c } else { @@ -338,7 +337,7 @@ impl FATFile { return Ok(()); } - let new_last_cluster = (new_size + fs.bytes_per_cluster() - 1) / fs.bytes_per_cluster(); + let new_last_cluster = new_size.div_ceil(fs.bytes_per_cluster()); if let Some(begin_delete) = fs.get_cluster_by_relative(self.first_cluster, new_last_cluster as usize) { @@ -463,8 +462,7 @@ impl FATDir { // 计算需要申请多少个簇 let clusters_required = - (remain_entries * FATRawDirEntry::DIR_ENTRY_LEN + fs.bytes_per_cluster() - 1) - / fs.bytes_per_cluster(); + (remain_entries * FATRawDirEntry::DIR_ENTRY_LEN).div_ceil(fs.bytes_per_cluster()); let mut first_cluster = Cluster::default(); let mut prev_cluster = current_cluster; // debug!( diff --git a/kernel/src/filesystem/fat/fs.rs b/kernel/src/filesystem/fat/fs.rs index 8e20210b5..fdd43859b 100644 --- a/kernel/src/filesystem/fat/fs.rs +++ b/kernel/src/filesystem/fat/fs.rs @@ -326,9 +326,8 @@ impl FATFileSystem { }; // 根目录项占用的扇区数（向上取整） - let root_dir_sectors: u64 = ((bpb.root_entries_cnt as u64 * 32) - + (bpb.bytes_per_sector as u64 - 1)) - / (bpb.bytes_per_sector as u64); + let root_dir_sectors: u64 = + (bpb.root_entries_cnt as u64 * 32).div_ceil(bpb.bytes_per_sector as u64); // FAT表大小（单位：扇区） let fat_size = if bpb.fat_size_16 != 0 { @@ -843,6 +842,7 @@ impl FATFileSystem { /// @return Ok(true) 正常 /// @return Ok(false) 不正常 /// @return Err(SystemError) 在判断时发生错误 + #[allow(dead_code)] pub fn is_hard_error_bit_ok(&mut self) -> Result { match self.bpb.fat_type { FATType::FAT32(_) => { @@ -935,10 +935,8 @@ impl FATFileSystem { _ => { // FAT12 / FAT16 - let root_dir_sectors: u64 = (((self.bpb.root_entries_cnt as u64) * 32) - + self.bpb.bytes_per_sector as u64 - - 1) - / self.bpb.bytes_per_sector as u64; + let root_dir_sectors: u64 = ((self.bpb.root_entries_cnt as u64) * 32) + .div_ceil(self.bpb.bytes_per_sector as u64); // 数据区扇区数 let data_sec: u64 = self.bpb.total_sectors_16 as u64 - (self.bpb.rsvd_sec_cnt as u64 @@ -1283,6 +1281,7 @@ impl FATFsInfo { /// @brief 根据fsinfo的信息，计算当前总的空闲簇数量 /// /// @param 当前文件系统的最大簇号 + #[allow(dead_code)] pub fn count_free_cluster(&self, max_cluster: Cluster) -> Option { let count_clusters = max_cluster.cluster_num - RESERVED_CLUSTERS as u64 + 1; // 信息不合理，当前的FsInfo中存储的free count大于计算出来的值 @@ -1301,6 +1300,7 @@ impl FATFsInfo { /// @brief 更新FsInfo中的“空闲簇统计信息“为new_count /// /// 请注意，除非手动调用`flush()`，否则本函数不会将数据刷入磁盘 + #[allow(dead_code)] pub fn update_free_count_abs(&mut self, new_count: u32) { self.free_count = new_count; } @@ -1308,6 +1308,7 @@ impl FATFsInfo { /// @brief 更新FsInfo中的“空闲簇统计信息“，把它加上delta. /// /// 请注意，除非手动调用`flush()`，否则本函数不会将数据刷入磁盘 + #[allow(dead_code)] pub fn update_free_count_delta(&mut self, delta: i32) { self.free_count = (self.free_count as i32 + delta) as u32; } @@ -1360,6 +1361,7 @@ impl FATFsInfo { /// @brief 读取磁盘上的Fs Info扇区，将里面的内容更新到结构体中 /// /// @param partition fs info所在的分区 + #[allow(dead_code)] pub fn update(&mut self, partition: Arc) -> Result<(), SystemError> { if let Some(off) = self.offset { let in_block_offset = off % LBA_SIZE as u64; @@ -1888,7 +1890,7 @@ struct ClusterIter<'a> { fs: &'a FATFileSystem, } -impl<'a> Iterator for ClusterIter<'a> { +impl Iterator for ClusterIter<'_> { type Item = Cluster; fn next(&mut self) -> Option { diff --git a/kernel/src/filesystem/mbr.rs b/kernel/src/filesystem/mbr.rs index 843be8959..9cf6eeddf 100644 --- a/kernel/src/filesystem/mbr.rs +++ b/kernel/src/filesystem/mbr.rs @@ -169,7 +169,7 @@ impl<'a> MbrPartitionIter<'a> { } } -impl<'a> Iterator for MbrPartitionIter<'a> { +impl Iterator for MbrPartitionIter<'_> { type Item = Partition; fn next(&mut self) -> Option { diff --git a/kernel/src/filesystem/overlayfs/mod.rs b/kernel/src/filesystem/overlayfs/mod.rs index 244217e32..3a68142ec 100644 --- a/kernel/src/filesystem/overlayfs/mod.rs +++ b/kernel/src/filesystem/overlayfs/mod.rs @@ -14,6 +14,7 @@ use alloc::sync::Arc; use alloc::sync::Weak; use alloc::vec::Vec; use entry::{OvlEntry, OvlLayer}; +use linkme::distributed_slice; use system_error::SystemError; const WHITEOUT_MODE: u64 = 0o020000 | 0o600; // whiteout字符设备文件模式与权限 @@ -89,7 +90,7 @@ struct OverlayFS { } #[derive(Debug)] -struct OvlInode { +pub struct OvlInode { redirect: String, // 重定向路径 file_type: FileType, flags: SpinLock, diff --git a/kernel/src/filesystem/procfs/syscall.rs b/kernel/src/filesystem/procfs/syscall.rs index f80365cc0..8066903c8 100644 --- a/kernel/src/filesystem/procfs/syscall.rs +++ b/kernel/src/filesystem/procfs/syscall.rs @@ -51,7 +51,6 @@ impl Syscall { /// - 成功，Ok(usize) /// - 失败，Err(SystemError) 操作失败，返回posix错误码 /// - pub fn do_syslog( syslog_action_type: usize, buf: &mut [u8], diff --git a/kernel/src/filesystem/vfs/file.rs b/kernel/src/filesystem/vfs/file.rs index 4b3d75b6f..3762474ea 100644 --- a/kernel/src/filesystem/vfs/file.rs +++ b/kernel/src/filesystem/vfs/file.rs @@ -800,7 +800,7 @@ impl<'a> FileDescriptorIterator<'a> { } } -impl<'a> Iterator for FileDescriptorIterator<'a> { +impl Iterator for FileDescriptorIterator<'_> { type Item = (i32, Arc); fn next(&mut self) -> Option { diff --git a/kernel/src/init/boot.rs b/kernel/src/init/boot.rs index b692e3fb1..ec345fd24 100644 --- a/kernel/src/init/boot.rs +++ b/kernel/src/init/boot.rs @@ -49,6 +49,7 @@ impl BootParams { core::str::from_utf8(&self.boot_cmdline()[..self.boot_cmdline_len()]).unwrap() } + #[allow(dead_code)] pub fn bootloader_name(&self) -> Option<&str> { self.bootloader_name.as_deref() } diff --git a/kernel/src/init/cmdline.rs b/kernel/src/init/cmdline.rs index 0d0ff040e..a313be51c 100644 --- a/kernel/src/init/cmdline.rs +++ b/kernel/src/init/cmdline.rs @@ -43,6 +43,7 @@ pub struct KernelCmdlineParamBuilder { inv: bool, } +#[allow(dead_code)] impl KernelCmdlineParamBuilder { pub const fn new(name: &'static str, ty: KCmdlineParamType) -> Self { Self { @@ -110,6 +111,7 @@ pub enum KernelCmdlineParameter { EarlyKV(&'static KernelCmdlineEarlyKV), } +#[allow(dead_code)] impl KernelCmdlineParameter { pub fn name(&self) -> &str { match self { @@ -195,6 +197,7 @@ pub struct KernelCmdlineEarlyKV { default: &'static str, } +#[allow(dead_code)] impl KernelCmdlineEarlyKV { pub const VALUE_MAX_LEN: usize = 256; diff --git a/kernel/src/ipc/signal_types.rs b/kernel/src/ipc/signal_types.rs index 6ebfbf611..d8d7b1677 100644 --- a/kernel/src/ipc/signal_types.rs +++ b/kernel/src/ipc/signal_types.rs @@ -297,7 +297,6 @@ pub struct UserSigaction { * siginfo中，根据signal的来源不同，该info中对应了不同的数据./= * 请注意，该info最大占用16字节 */ - #[repr(C)] #[derive(Copy, Clone, Debug)] pub struct SigInfo { diff --git a/kernel/src/lib.rs b/kernel/src/lib.rs index 881f6d319..c35cc12bc 100644 --- a/kernel/src/lib.rs +++ b/kernel/src/lib.rs @@ -2,20 +2,15 @@ #![feature(alloc_error_handler)] #![feature(allocator_api)] #![feature(arbitrary_self_types)] -#![feature(asm_const)] #![feature(concat_idents)] #![feature(const_for)] -#![feature(const_mut_refs)] -#![feature(const_option)] #![feature(const_trait_impl)] -#![feature(const_refs_to_cell)] #![feature(core_intrinsics)] #![feature(c_void_variant)] #![feature(extract_if)] #![feature(fn_align)] #![feature(linked_list_retain)] #![feature(naked_functions)] -#![feature(new_uninit)] #![feature(ptr_internals)] #![feature(trait_upcasting)] #![feature(slice_ptr_get)] @@ -23,16 +18,19 @@ #![feature(vec_into_raw_parts)] #![feature(c_variadic)] #![cfg_attr(target_os = "none", no_std)] -#![allow(internal_features)] +#![allow(static_mut_refs, non_local_definitions, internal_features)] // clippy的配置 #![deny(clippy::all)] -#![allow(clippy::bad_bit_mask)] // DragonOS允许在函数中使用return语句（尤其是长函数时，我们推荐这么做） -#![allow(clippy::let_and_return)] -#![allow(clippy::needless_pass_by_ref_mut)] -#![allow(clippy::needless_return)] -#![allow(clippy::single_char_pattern)] -#![allow(clippy::upper_case_acronyms)] +#![allow( + clippy::macro_metavars_in_unsafe, + clippy::upper_case_acronyms, + clippy::single_char_pattern, + clippy::needless_return, + clippy::needless_pass_by_ref_mut, + clippy::let_and_return, + clippy::bad_bit_mask +)] #[cfg(test)] #[macro_use] diff --git a/kernel/src/libs/casting.rs b/kernel/src/libs/casting.rs index a80dc2619..05c9911f5 100644 --- a/kernel/src/libs/casting.rs +++ b/kernel/src/libs/casting.rs @@ -47,7 +47,6 @@ use alloc::sync::Arc; /// /// fn test() { /// let a = A { name: "a".to_string() }; - /// let a_arc: Arc = Arc::new(a) as Arc; /// let a_arc2: Option> = a_arc.downcast_arc::(); /// assert!(a_arc2.is_some()); diff --git a/kernel/src/libs/cpumask.rs b/kernel/src/libs/cpumask.rs index fc86b860d..6bbf3917e 100644 --- a/kernel/src/libs/cpumask.rs +++ b/kernel/src/libs/cpumask.rs @@ -129,7 +129,7 @@ pub struct CpuMaskIter<'a> { begin: bool, } -impl<'a> Iterator for CpuMaskIter<'a> { +impl Iterator for CpuMaskIter<'_> { type Item = ProcessorId; fn next(&mut self) -> Option { diff --git a/kernel/src/libs/futex/futex.rs b/kernel/src/libs/futex/futex.rs index a3a26d7db..cc651ad2e 100644 --- a/kernel/src/libs/futex/futex.rs +++ b/kernel/src/libs/futex/futex.rs @@ -844,7 +844,7 @@ impl<'a> FutexIterator<'a> { } } -impl<'a> Iterator for FutexIterator<'a> { +impl Iterator for FutexIterator<'_> { type Item = VirtAddr; fn next(&mut self) -> Option { diff --git a/kernel/src/libs/lib_ui/screen_manager.rs b/kernel/src/libs/lib_ui/screen_manager.rs index 839ce3600..93f6788d5 100644 --- a/kernel/src/libs/lib_ui/screen_manager.rs +++ b/kernel/src/libs/lib_ui/screen_manager.rs @@ -298,7 +298,6 @@ pub fn scm_init(enable_put_to_window: bool) { /// ## 参数 /// /// - framework 要启动的ui框架 - pub fn scm_framework_enable(framework: Arc) -> Result { // 获取信息 let metadata = framework.metadata()?; @@ -321,7 +320,6 @@ pub fn scm_framework_enable(framework: Arc) -> Result) -> Result { // 把ui框架加入链表 diff --git a/kernel/src/libs/lib_ui/textui.rs b/kernel/src/libs/lib_ui/textui.rs index 16e523979..69558b6e6 100644 --- a/kernel/src/libs/lib_ui/textui.rs +++ b/kernel/src/libs/lib_ui/textui.rs @@ -595,7 +595,6 @@ impl TextuiWindow { /// -flags 标志位 /// -vlines_num 虚拟行的总数 /// -chars_num 每行最大的字符数 - pub fn new(flags: WindowFlag, vlines_num: i32, chars_num: i32) -> Self { let mut initial_vlines = Vec::new(); @@ -667,10 +666,8 @@ impl TextuiWindow { /// 重新渲染某个窗口的某个虚拟行 /// ## 参数 - /// - window 窗口结构体 /// - vline_id 虚拟行号 - fn textui_refresh_vline(&mut self, vline_id: LineId) -> Result<(), SystemError> { if self.flags.contains(WindowFlag::TEXTUI_CHROMATIC) { return self.textui_refresh_characters( @@ -788,12 +785,10 @@ impl TextuiWindow { } /// 根据输入的一个字符在窗口上输出 /// ## 参数 - /// - window 窗口 /// - character 字符 /// - FRcolor 前景色（RGB） /// - BKcolor 背景色（RGB） - fn textui_putchar_window( &mut self, character: char, @@ -1027,7 +1022,6 @@ where /// - character 字符 /// - FRcolor 前景色（RGB） /// - BKcolor 背景色（RGB） - #[no_mangle] pub extern "C" fn rs_textui_putchar(character: u8, fr_color: u32, bk_color: u32) -> i32 { if let Some(current_vc) = vc_manager().current_vc() { diff --git a/kernel/src/libs/rbtree.rs b/kernel/src/libs/rbtree.rs index f1fed00f0..6aedb791f 100644 --- a/kernel/src/libs/rbtree.rs +++ b/kernel/src/libs/rbtree.rs @@ -286,7 +286,6 @@ impl NodePtr { /// A red black tree implemented with Rust /// It is required that the keys implement the [`Ord`] traits. - /// # Examples /// ```rust /// use rbtree::RBTree; @@ -431,7 +430,7 @@ where impl Eq for RBTree {} -impl<'a, K: Ord + Debug, V: Debug> Index<&'a K> for RBTree { +impl Index<&K> for RBTree { type Output = V; #[inline] @@ -482,7 +481,7 @@ impl<'a, K: Ord + Debug, V: Debug> Clone for Keys<'a, K, V> { } } -impl<'a, K: Ord + Debug, V: Debug> fmt::Debug for Keys<'a, K, V> { +impl fmt::Debug for Keys<'_, K, V> { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { f.debug_list().entries(self.clone()).finish() } @@ -527,7 +526,7 @@ impl<'a, K: Ord + Debug, V: Debug> Clone for Values<'a, K, V> { } } -impl<'a, K: Ord + Debug, V: Debug> fmt::Debug for Values<'a, K, V> { +impl fmt::Debug for Values<'_, K, V> { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { f.debug_list().entries(self.clone()).finish() } @@ -575,7 +574,7 @@ impl<'a, K: Ord + Debug, V: Debug> Clone for ValuesMut<'a, K, V> { } } -impl<'a, K: Ord + Debug, V: Debug> fmt::Debug for ValuesMut<'a, K, V> { +impl fmt::Debug for ValuesMut<'_, K, V> { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { f.debug_list().entries(self.clone()).finish() } @@ -1651,7 +1650,7 @@ mod tests { let vec = vec![(1, 1), (2, 2), (3, 3)]; let mut map: RBTree<_, _> = vec.into_iter().collect(); for value in map.values_mut() { - *value = (*value) * 2 + *value *= 2 } let values: Vec<_> = map.values().cloned().collect(); assert_eq!(values.len(), 3); @@ -1808,7 +1807,7 @@ mod tests { b.insert(2, "two"); b.insert(3, "three"); - a.extend(b.into_iter()); + a.extend(b); assert_eq!(a.len(), 3); assert_eq!(a[&1], "one"); @@ -1818,6 +1817,7 @@ mod tests { #[test] fn test_rev_iter() { + use crate::libs::rbtree::RBTree; let mut a = RBTree::new(); a.insert(1, 1); a.insert(2, 2); @@ -1826,7 +1826,7 @@ mod tests { assert_eq!(a.len(), 3); let mut cache = vec![]; for e in a.iter().rev() { - cache.push(e.0.clone()); + cache.push(*e.0); } assert_eq!(&cache, &vec![3, 2, 1]); } diff --git a/kernel/src/libs/rwlock.rs b/kernel/src/libs/rwlock.rs index 63af797fc..2fabd5c2d 100644 --- a/kernel/src/libs/rwlock.rs +++ b/kernel/src/libs/rwlock.rs @@ -16,7 +16,6 @@ use crate::{ }; ///RwLock读写锁 - /// @brief READER位占据从右往左数第三个比特位 const READER: u32 = 1 << 2; @@ -548,7 +547,7 @@ impl<'rwlock, T> RwLockWriteGuard<'rwlock, T> { } } -impl<'rwlock, T> Deref for RwLockReadGuard<'rwlock, T> { +impl Deref for RwLockReadGuard<'_, T> { type Target = T; fn deref(&self) -> &Self::Target { @@ -556,7 +555,7 @@ impl<'rwlock, T> Deref for RwLockReadGuard<'rwlock, T> { } } -impl<'rwlock, T> Deref for RwLockUpgradableGuard<'rwlock, T> { +impl Deref for RwLockUpgradableGuard<'_, T> { type Target = T; fn deref(&self) -> &Self::Target { @@ -564,7 +563,7 @@ impl<'rwlock, T> Deref for RwLockUpgradableGuard<'rwlock, T> { } } -impl<'rwlock, T> Deref for RwLockWriteGuard<'rwlock, T> { +impl Deref for RwLockWriteGuard<'_, T> { type Target = T; fn deref(&self) -> &Self::Target { @@ -572,13 +571,13 @@ impl<'rwlock, T> Deref for RwLockWriteGuard<'rwlock, T> { } } -impl<'rwlock, T> DerefMut for RwLockWriteGuard<'rwlock, T> { +impl DerefMut for RwLockWriteGuard<'_, T> { fn deref_mut(&mut self) -> &mut Self::Target { return unsafe { &mut *self.data }; } } -impl<'rwlock, T> Drop for RwLockReadGuard<'rwlock, T> { +impl Drop for RwLockReadGuard<'_, T> { fn drop(&mut self) { debug_assert!(self.lock.load(Ordering::Relaxed) & !(WRITER | UPGRADED) > 0); self.lock.fetch_sub(READER, Ordering::Release); @@ -586,7 +585,7 @@ impl<'rwlock, T> Drop for RwLockReadGuard<'rwlock, T> { } } -impl<'rwlock, T> Drop for RwLockUpgradableGuard<'rwlock, T> { +impl Drop for RwLockUpgradableGuard<'_, T> { fn drop(&mut self) { debug_assert_eq!( self.inner.lock.load(Ordering::Relaxed) & (WRITER | UPGRADED), @@ -598,7 +597,7 @@ impl<'rwlock, T> Drop for RwLockUpgradableGuard<'rwlock, T> { } } -impl<'rwlock, T> Drop for RwLockWriteGuard<'rwlock, T> { +impl Drop for RwLockWriteGuard<'_, T> { fn drop(&mut self) { debug_assert_eq!(self.inner.lock.load(Ordering::Relaxed) & WRITER, WRITER); self.inner diff --git a/kernel/src/libs/volatile.rs b/kernel/src/libs/volatile.rs index efb8f3f33..e967be07a 100644 --- a/kernel/src/libs/volatile.rs +++ b/kernel/src/libs/volatile.rs @@ -64,7 +64,6 @@ macro_rules! volatile_write_bit { /// volwrite!(self.common_cfg, queue_enable, 0); /// /// 这样做不仅使代码的可读性提高了，也避免了对只读寄存器进行写入的误操作 - /// 只读寄存器 #[derive(Default)] #[repr(transparent)] diff --git a/kernel/src/mm/memblock.rs b/kernel/src/mm/memblock.rs index d17bd3434..ae5861463 100644 --- a/kernel/src/mm/memblock.rs +++ b/kernel/src/mm/memblock.rs @@ -473,7 +473,7 @@ pub struct MemBlockIter<'a> { } #[allow(dead_code)] -impl<'a> MemBlockIter<'a> { +impl MemBlockIter<'_> { /// 获取内存区域数量 pub fn total_num(&self) -> usize { self.inner.initial_memory_regions_num @@ -490,7 +490,7 @@ impl<'a> MemBlockIter<'a> { } } -impl<'a> Iterator for MemBlockIter<'a> { +impl Iterator for MemBlockIter<'_> { type Item = PhysMemoryArea; fn next(&mut self) -> Option { diff --git a/kernel/src/perf/bpf.rs b/kernel/src/perf/bpf.rs index 84af5436c..b283b2e54 100644 --- a/kernel/src/perf/bpf.rs +++ b/kernel/src/perf/bpf.rs @@ -208,6 +208,8 @@ impl RingPage { let data_head = unsafe { &(*(self.ptr as *mut perf_event_mmap_page)).data_head }; data_tail != data_head } + + #[allow(dead_code)] pub fn as_slice(&self) -> &[u8] { unsafe { core::slice::from_raw_parts(self.ptr as *const u8, self.size) } } diff --git a/kernel/src/perf/util.rs b/kernel/src/perf/util.rs index b1cc43eb4..1b24eed6b 100644 --- a/kernel/src/perf/util.rs +++ b/kernel/src/perf/util.rs @@ -116,7 +116,7 @@ pub struct PerfSample<'a> { pub value: &'a [u8], } -impl<'a> PerfSample<'a> { +impl PerfSample<'_> { pub fn calculate_size(value_size: usize) -> usize { size_of::() + value_size } diff --git a/kernel/src/process/exit.rs b/kernel/src/process/exit.rs index c88a9493b..9420c0fd0 100644 --- a/kernel/src/process/exit.rs +++ b/kernel/src/process/exit.rs @@ -39,7 +39,7 @@ pub struct WaitIdInfo { pub cause: i32, } -impl<'a> KernelWaitOption<'a> { +impl KernelWaitOption<'_> { pub fn new(pid_type: PidType, pid: Pid, options: WaitOption) -> Self { Self { pid_type, diff --git a/kernel/src/process/syscall.rs b/kernel/src/process/syscall.rs index 9dd04a114..9ba37e829 100644 --- a/kernel/src/process/syscall.rs +++ b/kernel/src/process/syscall.rs @@ -212,7 +212,7 @@ impl Syscall { return Ok(target_proc.basic().pgid()); } /// @brief 获取当前进程的父进程id - + /// /// 若为initproc则ppid设置为0 pub fn getppid() -> Result { let current_pcb = ProcessManager::current_pcb(); diff --git a/kernel/src/syscall/misc.rs b/kernel/src/syscall/misc.rs index f78029291..92879b3eb 100644 --- a/kernel/src/syscall/misc.rs +++ b/kernel/src/syscall/misc.rs @@ -10,12 +10,11 @@ use system_error::SystemError; use super::{user_access::UserBufferWriter, Syscall}; -#[repr(C)] - /// 系统信息 /// /// 参考 https://code.dragonos.org.cn/xref/linux-6.1.9/include/uapi/linux/sysinfo.h#8 #[derive(Debug, Default, Copy, Clone)] +#[repr(C)] pub struct SysInfo { uptime: u64, loads: [u64; 3], diff --git a/kernel/src/syscall/user_access.rs b/kernel/src/syscall/user_access.rs index 18a74764e..2850dfdc7 100644 --- a/kernel/src/syscall/user_access.rs +++ b/kernel/src/syscall/user_access.rs @@ -160,7 +160,7 @@ pub struct UserBufferReader<'a> { } #[allow(dead_code)] -impl<'a> UserBufferReader<'a> { +impl UserBufferReader<'_> { /// 构造一个指向用户空间位置的BufferReader，为了兼容类似传入 *const u8 的情况，使用单独的泛型来进行初始化 /// /// @param addr 用户空间指针 @@ -321,7 +321,7 @@ impl<'a> UserBufferWriter<'a> { return Ok(()); } - pub fn buffer(&'a mut self, offset: usize) -> Result<&mut [T], SystemError> { + pub fn buffer(&'a mut self, offset: usize) -> Result<&'a mut [T], SystemError> { Self::convert_with_offset::(self.buffer, offset).map_err(|_| SystemError::EINVAL) } diff --git a/tools/BUILD_CONTAINER_VERSION b/tools/BUILD_CONTAINER_VERSION index 59b6ef75f..536385712 100644 --- a/tools/BUILD_CONTAINER_VERSION +++ b/tools/BUILD_CONTAINER_VERSION @@ -1 +1 @@ -v1.5 \ No newline at end of file +v1.6 \ No newline at end of file diff --git a/tools/Makefile b/tools/Makefile index c4faeea5f..94ff2833a 100644 --- a/tools/Makefile +++ b/tools/Makefile @@ -6,4 +6,4 @@ clean: @cargo clean check: - @cargo +nightly-2024-07-23 check --workspace --message-format=json + @cargo +nightly-2024-11-05 check --workspace --message-format=json diff --git a/tools/bootstrap.sh b/tools/bootstrap.sh index ddb5a6088..b859c3ae9 100644 --- a/tools/bootstrap.sh +++ b/tools/bootstrap.sh @@ -23,7 +23,7 @@ DEFAULT_INSTALL="false" export RUSTUP_DIST_SERVER=${RUSTUP_DIST_SERVER:-https://rsproxy.cn} export RUSTUP_UPDATE_ROOT=${RUSTUP_UPDATE_ROOT:-https://rsproxy.cn/rustup} -export RUST_VERSION="${RUST_VERSION:-nightly-2024-07-23}" +export RUST_VERSION="${RUST_VERSION:-nightly-2024-11-05}" banner() { @@ -233,21 +233,21 @@ rustInstall() { echo "正在安装DragonOS所需的rust组件...首次安装需要一些时间来更新索引，请耐心等待..." cargo install cargo-binutils cargo install bpf-linker - rustup toolchain install nightly-2023-08-15-x86_64-unknown-linux-gnu + rustup toolchain install nightly-2024-11-05-x86_64-unknown-linux-gnu rustup toolchain install $RUST_VERSION-x86_64-unknown-linux-gnu rustup component add rust-src --toolchain $RUST_VERSION-x86_64-unknown-linux-gnu - rustup component add rust-src --toolchain nightly-2023-08-15-x86_64-unknown-linux-gnu + rustup component add rust-src --toolchain nightly-2024-11-05-x86_64-unknown-linux-gnu rustup target add x86_64-unknown-none --toolchain $RUST_VERSION-x86_64-unknown-linux-gnu - rustup target add x86_64-unknown-none --toolchain nightly-2023-08-15-x86_64-unknown-linux-gnu - rustup target add x86_64-unknown-linux-musl --toolchain nightly-2023-08-15-x86_64-unknown-linux-gnu + rustup target add x86_64-unknown-none --toolchain nightly-2024-11-05-x86_64-unknown-linux-gnu + rustup target add x86_64-unknown-linux-musl --toolchain nightly-2024-11-05-x86_64-unknown-linux-gnu rustup target add x86_64-unknown-linux-musl --toolchain $RUST_VERSION-x86_64-unknown-linux-gnu rustup toolchain install $RUST_VERSION-riscv64gc-unknown-linux-gnu --force-non-host - rustup toolchain install nightly-2023-08-15-riscv64gc-unknown-linux-gnu --force-non-host + rustup toolchain install nightly-2024-11-05-riscv64gc-unknown-linux-gnu --force-non-host rustup target add riscv64gc-unknown-none-elf --toolchain $RUST_VERSION-riscv64gc-unknown-linux-gnu rustup target add riscv64imac-unknown-none-elf --toolchain $RUST_VERSION-riscv64gc-unknown-linux-gnu - rustup target add riscv64gc-unknown-none-elf --toolchain nightly-2023-08-15-riscv64gc-unknown-linux-gnu - rustup target add riscv64imac-unknown-none-elf --toolchain nightly-2023-08-15-riscv64gc-unknown-linux-gnu + rustup target add riscv64gc-unknown-none-elf --toolchain nightly-2024-11-05-riscv64gc-unknown-linux-gnu + rustup target add riscv64imac-unknown-none-elf --toolchain nightly-2024-11-05-riscv64gc-unknown-linux-gnu rustup component add rust-src --toolchain nightly-x86_64-unknown-linux-gnu rustup component add rust-src diff --git a/tools/debugging/logmonitor/src/app.rs b/tools/debugging/logmonitor/src/app.rs index ab652a0b2..92d485665 100644 --- a/tools/debugging/logmonitor/src/app.rs +++ b/tools/debugging/logmonitor/src/app.rs @@ -110,7 +110,7 @@ pub struct TabsState<'a> { } impl<'a> TabsState<'a> { - pub fn new(titles: Vec<&'a str>) -> TabsState { + pub fn new(titles: Vec<&'a str>) -> TabsState<'a> { TabsState { titles, index: 0 } } pub fn next(&mut self) { diff --git a/user/apps/clear/Makefile b/user/apps/clear/Makefile index 127c6ccb3..0d1403539 100644 --- a/user/apps/clear/Makefile +++ b/user/apps/clear/Makefile @@ -1,4 +1,4 @@ -TOOLCHAIN="+nightly-2024-07-23-x86_64-unknown-linux-gnu" +TOOLCHAIN="+nightly-2024-11-05-x86_64-unknown-linux-gnu" # RUSTFLAGS+="-C target-feature=+crt-static -C link-arg=-no-pie" ifdef DADK_CURRENT_BUILD_DIR diff --git a/user/apps/test-backlog/Makefile b/user/apps/test-backlog/Makefile index f65b0d520..99b095d51 100644 --- a/user/apps/test-backlog/Makefile +++ b/user/apps/test-backlog/Makefile @@ -1,4 +1,4 @@ -TOOLCHAIN="+nightly-2024-07-23-x86_64-unknown-linux-gnu" +TOOLCHAIN="+nightly-2024-11-05-x86_64-unknown-linux-gnu" RUSTFLAGS+="" ifdef DADK_CURRENT_BUILD_DIR diff --git a/user/apps/test-blockcache/Makefile b/user/apps/test-blockcache/Makefile index 86a122015..e9aa93ae8 100644 --- a/user/apps/test-blockcache/Makefile +++ b/user/apps/test-blockcache/Makefile @@ -1,6 +1,6 @@ # The toolchain we use. # You can get it by running DragonOS' `tools/bootstrap.sh` -TOOLCHAIN="+nightly-2024-07-23-x86_64-unknown-linux-gnu" +TOOLCHAIN="+nightly-2024-11-05-x86_64-unknown-linux-gnu" RUSTFLAGS+="-C target-feature=+crt-static -C link-arg=-no-pie" # 如果是在dadk中编译，那么安装到dadk的安装目录中 diff --git a/user/apps/test-for-robustfutex/Makefile b/user/apps/test-for-robustfutex/Makefile index d9eb6cd1e..352c0562c 100644 --- a/user/apps/test-for-robustfutex/Makefile +++ b/user/apps/test-for-robustfutex/Makefile @@ -1,4 +1,4 @@ -TOOLCHAIN="+nightly-2024-07-23-x86_64-unknown-linux-gnu" +TOOLCHAIN="+nightly-2024-11-05-x86_64-unknown-linux-gnu" RUSTFLAGS+="" ifdef DADK_CURRENT_BUILD_DIR diff --git a/user/apps/test-mount/Makefile b/user/apps/test-mount/Makefile index d9eb6cd1e..352c0562c 100644 --- a/user/apps/test-mount/Makefile +++ b/user/apps/test-mount/Makefile @@ -1,4 +1,4 @@ -TOOLCHAIN="+nightly-2024-07-23-x86_64-unknown-linux-gnu" +TOOLCHAIN="+nightly-2024-11-05-x86_64-unknown-linux-gnu" RUSTFLAGS+="" ifdef DADK_CURRENT_BUILD_DIR diff --git a/user/apps/test_alarm/Makefile b/user/apps/test_alarm/Makefile index d9eb6cd1e..352c0562c 100644 --- a/user/apps/test_alarm/Makefile +++ b/user/apps/test_alarm/Makefile @@ -1,4 +1,4 @@ -TOOLCHAIN="+nightly-2024-07-23-x86_64-unknown-linux-gnu" +TOOLCHAIN="+nightly-2024-11-05-x86_64-unknown-linux-gnu" RUSTFLAGS+="" ifdef DADK_CURRENT_BUILD_DIR diff --git a/user/apps/test_ebpf/Makefile b/user/apps/test_ebpf/Makefile index f2770a320..0b5d9e43a 100644 --- a/user/apps/test_ebpf/Makefile +++ b/user/apps/test_ebpf/Makefile @@ -1,4 +1,4 @@ -TOOLCHAIN="+nightly-2024-07-23-x86_64-unknown-linux-gnu" +TOOLCHAIN="+nightly-2024-11-05-x86_64-unknown-linux-gnu" RUSTFLAGS+="" ifdef DADK_CURRENT_BUILD_DIR diff --git a/user/apps/test_ebpf/syscall_ebpf/syscall_ebpf-ebpf/rust-toolchain.toml b/user/apps/test_ebpf/syscall_ebpf/syscall_ebpf-ebpf/rust-toolchain.toml index 447b8f049..fda4ec82e 100644 --- a/user/apps/test_ebpf/syscall_ebpf/syscall_ebpf-ebpf/rust-toolchain.toml +++ b/user/apps/test_ebpf/syscall_ebpf/syscall_ebpf-ebpf/rust-toolchain.toml @@ -1,5 +1,5 @@ [toolchain] -channel = "nightly-2024-07-23" +channel = "nightly-2024-11-05" # The source code of rustc, provided by the rust-src component, is needed for # building eBPF programs. components = [ diff --git a/user/apps/test_namespace/Makefile b/user/apps/test_namespace/Makefile index 1b0274d20..352c0562c 100644 --- a/user/apps/test_namespace/Makefile +++ b/user/apps/test_namespace/Makefile @@ -1,4 +1,4 @@ -TOOLCHAIN="+nightly-2023-08-15-x86_64-unknown-linux-gnu" +TOOLCHAIN="+nightly-2024-11-05-x86_64-unknown-linux-gnu" RUSTFLAGS+="" ifdef DADK_CURRENT_BUILD_DIR diff --git a/user/apps/test_namespace/makefile.toml b/user/apps/test_namespace/makefile.toml index dc3f667e6..309c8631d 100644 --- a/user/apps/test_namespace/makefile.toml +++ b/user/apps/test_namespace/makefile.toml @@ -1,7 +1,7 @@ # Makefile.toml [env] -TOOLCHAIN = "+nightly-2023-08-15-x86_64-unknown-linux-gnu" +TOOLCHAIN = "+nightly-2024-11-05-x86_64-unknown-linux-gnu" ARCH = { default = "x86_64" } RUST_TARGET = { default = { if = "eq(env.ARCH, 'riscv64')", value = "riscv64gc-unknown-linux-gnu", else = "x86_64-unknown-linux-musl" } } diff --git a/user/apps/test_socket/Makefile b/user/apps/test_socket/Makefile index d9eb6cd1e..352c0562c 100644 --- a/user/apps/test_socket/Makefile +++ b/user/apps/test_socket/Makefile @@ -1,4 +1,4 @@ -TOOLCHAIN="+nightly-2024-07-23-x86_64-unknown-linux-gnu" +TOOLCHAIN="+nightly-2024-11-05-x86_64-unknown-linux-gnu" RUSTFLAGS+="" ifdef DADK_CURRENT_BUILD_DIR diff --git a/user/apps/test_statx/Makefile b/user/apps/test_statx/Makefile index 127c6ccb3..0d1403539 100644 --- a/user/apps/test_statx/Makefile +++ b/user/apps/test_statx/Makefile @@ -1,4 +1,4 @@ -TOOLCHAIN="+nightly-2024-07-23-x86_64-unknown-linux-gnu" +TOOLCHAIN="+nightly-2024-11-05-x86_64-unknown-linux-gnu" # RUSTFLAGS+="-C target-feature=+crt-static -C link-arg=-no-pie" ifdef DADK_CURRENT_BUILD_DIR diff --git a/user/apps/test_tokio/Makefile b/user/apps/test_tokio/Makefile index d9eb6cd1e..352c0562c 100644 --- a/user/apps/test_tokio/Makefile +++ b/user/apps/test_tokio/Makefile @@ -1,4 +1,4 @@ -TOOLCHAIN="+nightly-2024-07-23-x86_64-unknown-linux-gnu" +TOOLCHAIN="+nightly-2024-11-05-x86_64-unknown-linux-gnu" RUSTFLAGS+="" ifdef DADK_CURRENT_BUILD_DIR diff --git a/user/apps/user-manage/Makefile b/user/apps/user-manage/Makefile index 82cc81f92..b1691ea47 100644 --- a/user/apps/user-manage/Makefile +++ b/user/apps/user-manage/Makefile @@ -1,6 +1,6 @@ # The toolchain we use. # You can get it by running DragonOS' `tools/bootstrap.sh` -TOOLCHAIN="+nightly-2024-07-23-x86_64-unknown-linux-gnu" +TOOLCHAIN="+nightly-2024-11-05-x86_64-unknown-linux-gnu" RUSTFLAGS+="" ifdef DADK_CURRENT_BUILD_DIR diff --git a/user/dadk/config/dog_0_1_0.dadk b/user/dadk/config/dog_0_1_0.dadk index 1e145150e..157f085cb 100644 --- a/user/dadk/config/dog_0_1_0.dadk +++ b/user/dadk/config/dog_0_1_0.dadk @@ -8,7 +8,7 @@ "Git": { "url": "https://git.mirrors.dragonos.org.cn/DragonOS-Community/dog.git", "branch": null, - "revision": "4ad6075686" + "revision": "6f2c0c8f12" } } }, diff --git a/user/dadk/config/dragon_reach-0.1.0.dadk b/user/dadk/config/dragon_reach-0.1.0.dadk index bf0472a34..73dac2ac2 100644 --- a/user/dadk/config/dragon_reach-0.1.0.dadk +++ b/user/dadk/config/dragon_reach-0.1.0.dadk @@ -6,7 +6,7 @@ "BuildFromSource": { "Git": { "url" : "https://git.mirrors.dragonos.org.cn/DragonOS-Community/DragonReach.git", - "revision": "01cdc56863" + "revision": "e945c217b3" } } }, From 0f094e50dee4cc20efbbe32852a4c79e619c4806 Mon Sep 17 00:00:00 2001 From: LoGin Date: Tue, 12 Nov 2024 14:01:26 +0800 Subject: [PATCH 27/30] =?UTF-8?q?fix:=20=E4=BF=AE=E5=A4=8D=E4=BA=86do=20ex?= =?UTF-8?q?ecve=20=E5=8A=A0=E8=BD=BD=E7=A8=8B=E5=BA=8F=E5=A4=B1=E8=B4=A5?= =?UTF-8?q?=E6=97=B6,=E6=B2=A1=E8=83=BD=E6=AD=A3=E7=A1=AE=E8=BF=94?= =?UTF-8?q?=E5=9B=9E=E9=94=99=E8=AF=AF=E7=A0=81=E7=BB=99=E7=94=A8=E6=88=B7?= =?UTF-8?q?=E6=80=81=E7=9A=84=E9=97=AE=E9=A2=98=20(#1042)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: 修复了do execve 加载程序失败时,没能正确返回错误码给用户态的问题 --- kernel/src/arch/riscv64/process/syscall.rs | 93 ++-------------- kernel/src/arch/x86_64/process/syscall.rs | 90 ++-------------- kernel/src/process/syscall.rs | 118 ++++++++++++++++++++- 3 files changed, 131 insertions(+), 170 deletions(-) diff --git a/kernel/src/arch/riscv64/process/syscall.rs b/kernel/src/arch/riscv64/process/syscall.rs index b60f347b4..21ddbed31 100644 --- a/kernel/src/arch/riscv64/process/syscall.rs +++ b/kernel/src/arch/riscv64/process/syscall.rs @@ -1,96 +1,21 @@ -use alloc::{ffi::CString, string::String, vec::Vec}; use riscv::register::sstatus::{FS, SPP}; use system_error::SystemError; use crate::{ - arch::{interrupt::TrapFrame, CurrentIrqArch}, - exception::InterruptArch, - mm::ucontext::AddressSpace, - process::{ - exec::{load_binary_file, ExecParam, ExecParamFlags}, - ProcessManager, - }, + arch::interrupt::TrapFrame, + mm::VirtAddr, + process::exec::{BinaryLoaderResult, ExecParam}, syscall::Syscall, }; impl Syscall { - pub fn do_execve( - path: String, - argv: Vec, - envp: Vec, + pub fn arch_do_execve( regs: &mut TrapFrame, + param: &ExecParam, + load_result: &BinaryLoaderResult, + user_sp: VirtAddr, + argv_ptr: VirtAddr, ) -> Result<(), SystemError> { - // 关中断，防止在设置地址空间的时候，发生中断，然后进调度器，出现错误。 - let irq_guard = unsafe { CurrentIrqArch::save_and_disable_irq() }; - let pcb = ProcessManager::current_pcb(); - // crate::debug!( - // "pid: {:?} do_execve: path: {:?}, argv: {:?}, envp: {:?}\n", - // pcb.pid(), - // path, - // argv, - // envp - // ); - - let mut basic_info = pcb.basic_mut(); - // 暂存原本的用户地址空间的引用(因为如果在切换页表之前释放了它，可能会造成内存use after free) - let old_address_space = basic_info.user_vm(); - - // 在pcb中原来的用户地址空间 - unsafe { - basic_info.set_user_vm(None); - } - // 创建新的地址空间并设置为当前地址空间 - let address_space = AddressSpace::new(true).expect("Failed to create new address space"); - unsafe { - basic_info.set_user_vm(Some(address_space.clone())); - } - - // to avoid deadlock - drop(basic_info); - - assert!( - AddressSpace::is_current(&address_space), - "Failed to set address space" - ); - // debug!("Switch to new address space"); - - // 切换到新的用户地址空间 - unsafe { address_space.read().user_mapper.utable.make_current() }; - - drop(old_address_space); - drop(irq_guard); - // debug!("to load binary file"); - let mut param = ExecParam::new(path.as_str(), address_space.clone(), ExecParamFlags::EXEC)?; - - // 加载可执行文件 - let load_result = load_binary_file(&mut param)?; - // debug!("load binary file done"); - // debug!("argv: {:?}, envp: {:?}", argv, envp); - param.init_info_mut().args = argv; - param.init_info_mut().envs = envp; - - // 把proc_init_info写到用户栈上 - let mut ustack_message = unsafe { - address_space - .write() - .user_stack_mut() - .expect("No user stack found") - .clone_info_only() - }; - let (user_sp, argv_ptr) = unsafe { - param - .init_info() - .push_at( - // address_space - // .write() - // .user_stack_mut() - // .expect("No user stack found"), - &mut ustack_message, - ) - .expect("Failed to push proc_init_info to user stack") - }; - address_space.write().user_stack = Some(ustack_message); - // debug!("write proc_init_info to user stack done"); regs.a0 = param.init_info().args.len(); @@ -104,8 +29,6 @@ impl Syscall { regs.status.update_fs(FS::Clean); regs.status.update_sum(true); - drop(param); - return Ok(()); } diff --git a/kernel/src/arch/x86_64/process/syscall.rs b/kernel/src/arch/x86_64/process/syscall.rs index d2652beb9..a10bd57de 100644 --- a/kernel/src/arch/x86_64/process/syscall.rs +++ b/kernel/src/arch/x86_64/process/syscall.rs @@ -1,99 +1,27 @@ -use alloc::{ffi::CString, string::String, sync::Arc, vec::Vec}; +use alloc::sync::Arc; use system_error::SystemError; use crate::{ arch::{ interrupt::TrapFrame, process::table::{USER_CS, USER_DS}, - CurrentIrqArch, }, - exception::InterruptArch, - mm::ucontext::AddressSpace, + mm::VirtAddr, process::{ - exec::{load_binary_file, ExecParam, ExecParamFlags}, + exec::{BinaryLoaderResult, ExecParam}, ProcessControlBlock, ProcessManager, }, syscall::{user_access::UserBufferWriter, Syscall}, }; impl Syscall { - pub fn do_execve( - path: String, - argv: Vec, - envp: Vec, + pub fn arch_do_execve( regs: &mut TrapFrame, + param: &ExecParam, + load_result: &BinaryLoaderResult, + user_sp: VirtAddr, + argv_ptr: VirtAddr, ) -> Result<(), SystemError> { - // 关中断，防止在设置地址空间的时候，发生中断，然后进调度器，出现错误。 - let irq_guard = unsafe { CurrentIrqArch::save_and_disable_irq() }; - let pcb = ProcessManager::current_pcb(); - // log::debug!( - // "pid: {:?} do_execve: path: {:?}, argv: {:?}, envp: {:?}\n", - // pcb.pid(), - // path, - // argv, - // envp - // ); - - let mut basic_info = pcb.basic_mut(); - // 暂存原本的用户地址空间的引用(因为如果在切换页表之前释放了它，可能会造成内存use after free) - let old_address_space = basic_info.user_vm(); - - // 在pcb中原来的用户地址空间 - unsafe { - basic_info.set_user_vm(None); - } - // 创建新的地址空间并设置为当前地址空间 - let address_space = AddressSpace::new(true).expect("Failed to create new address space"); - unsafe { - basic_info.set_user_vm(Some(address_space.clone())); - } - - // to avoid deadlock - drop(basic_info); - - assert!( - AddressSpace::is_current(&address_space), - "Failed to set address space" - ); - // debug!("Switch to new address space"); - - // 切换到新的用户地址空间 - unsafe { address_space.read().user_mapper.utable.make_current() }; - - drop(old_address_space); - drop(irq_guard); - // debug!("to load binary file"); - let mut param = ExecParam::new(path.as_str(), address_space.clone(), ExecParamFlags::EXEC)?; - - // 加载可执行文件 - let load_result = load_binary_file(&mut param)?; - // debug!("load binary file done"); - // debug!("argv: {:?}, envp: {:?}", argv, envp); - param.init_info_mut().args = argv; - param.init_info_mut().envs = envp; - - // 把proc_init_info写到用户栈上 - let mut ustack_message = unsafe { - address_space - .write() - .user_stack_mut() - .expect("No user stack found") - .clone_info_only() - }; - let (user_sp, argv_ptr) = unsafe { - param - .init_info() - .push_at( - // address_space - // .write() - // .user_stack_mut() - // .expect("No user stack found"), - &mut ustack_message, - ) - .expect("Failed to push proc_init_info to user stack") - }; - address_space.write().user_stack = Some(ustack_message); - // debug!("write proc_init_info to user stack done"); // （兼容旧版libc）把argv的指针写到寄存器内 @@ -114,8 +42,6 @@ impl Syscall { regs.rflags = 0x200; regs.rax = 1; - drop(param); - // debug!("regs: {:?}\n", regs); // crate::debug!( diff --git a/kernel/src/process/syscall.rs b/kernel/src/process/syscall.rs index 9ba37e829..ddcb96202 100644 --- a/kernel/src/process/syscall.rs +++ b/kernel/src/process/syscall.rs @@ -1,24 +1,34 @@ use core::ffi::c_void; -use alloc::{ffi::CString, string::ToString, sync::Arc, vec::Vec}; +use alloc::{ + ffi::CString, + string::{String, ToString}, + sync::Arc, + vec::Vec, +}; use log::error; use system_error::SystemError; use super::{ abi::WaitOption, cred::{Kgid, Kuid}, + exec::{load_binary_file, ExecParam, ExecParamFlags}, exit::kernel_wait4, fork::{CloneFlags, KernelCloneArgs}, resource::{RLimit64, RLimitID, RUsage, RUsageWho}, KernelStack, Pid, ProcessManager, }; use crate::{ - arch::{interrupt::TrapFrame, MMArch}, + arch::{interrupt::TrapFrame, CurrentIrqArch, MMArch}, + exception::InterruptArch, filesystem::{ procfs::procfs_register_pid, vfs::{file::FileDescriptorVec, MAX_PATHLEN}, }, - mm::{ucontext::UserStack, verify_area, MemoryManagementArch, VirtAddr}, + mm::{ + ucontext::{AddressSpace, UserStack}, + verify_area, MemoryManagementArch, VirtAddr, + }, process::ProcessControlBlock, sched::completion::Completion, syscall::{ @@ -139,6 +149,54 @@ impl Syscall { return Ok(()); } + pub fn do_execve( + path: String, + argv: Vec, + envp: Vec, + regs: &mut TrapFrame, + ) -> Result<(), SystemError> { + let address_space = AddressSpace::new(true).expect("Failed to create new address space"); + // debug!("to load binary file"); + let mut param = ExecParam::new(path.as_str(), address_space.clone(), ExecParamFlags::EXEC)?; + let old_vm = do_execve_switch_user_vm(address_space.clone()); + + // 加载可执行文件 + let load_result = load_binary_file(&mut param).inspect_err(|_| { + if let Some(old_vm) = old_vm { + do_execve_switch_user_vm(old_vm); + } + })?; + + // debug!("load binary file done"); + // debug!("argv: {:?}, envp: {:?}", argv, envp); + param.init_info_mut().args = argv; + param.init_info_mut().envs = envp; + + // 把proc_init_info写到用户栈上 + let mut ustack_message = unsafe { + address_space + .write() + .user_stack_mut() + .expect("No user stack found") + .clone_info_only() + }; + let (user_sp, argv_ptr) = unsafe { + param + .init_info() + .push_at( + // address_space + // .write() + // .user_stack_mut() + // .expect("No user stack found"), + &mut ustack_message, + ) + .expect("Failed to push proc_init_info to user stack") + }; + address_space.write().user_stack = Some(ustack_message); + + Self::arch_do_execve(regs, ¶m, &load_result, user_sp, argv_ptr) + } + pub fn wait4( pid: i64, wstatus: *mut i32, @@ -499,3 +557,57 @@ impl Syscall { return Ok(0); } } + +/// 切换用户虚拟内存空间 +/// +/// 该函数用于在执行系统调用 `execve` 时切换用户进程的虚拟内存空间。 +/// +/// # 参数 +/// - `new_vm`: 新的用户地址空间，类型为 `Arc`。 +/// +/// # 返回值 +/// - 返回旧的用户地址空间的引用，类型为 `Option>`。 +/// +/// # 错误处理 +/// 如果地址空间切换失败，函数会触发断言失败，并输出错误信息。 +fn do_execve_switch_user_vm(new_vm: Arc) -> Option> { + // 关中断，防止在设置地址空间的时候，发生中断，然后进调度器，出现错误。 + let irq_guard = unsafe { CurrentIrqArch::save_and_disable_irq() }; + let pcb = ProcessManager::current_pcb(); + // log::debug!( + // "pid: {:?} do_execve: path: {:?}, argv: {:?}, envp: {:?}\n", + // pcb.pid(), + // path, + // argv, + // envp + // ); + + let mut basic_info = pcb.basic_mut(); + // 暂存原本的用户地址空间的引用(因为如果在切换页表之前释放了它，可能会造成内存use after free) + let old_address_space = basic_info.user_vm(); + + // 在pcb中原来的用户地址空间 + unsafe { + basic_info.set_user_vm(None); + } + // 创建新的地址空间并设置为当前地址空间 + unsafe { + basic_info.set_user_vm(Some(new_vm.clone())); + } + + // to avoid deadlock + drop(basic_info); + + assert!( + AddressSpace::is_current(&new_vm), + "Failed to set address space" + ); + // debug!("Switch to new address space"); + + // 切换到新的用户地址空间 + unsafe { new_vm.read().user_mapper.utable.make_current() }; + + drop(irq_guard); + + old_address_space +} From ff76b0fcb886af9d15e3ef1c1bb61122541f6b75 Mon Sep 17 00:00:00 2001 From: MemoryShore <1353318529@qq.com> Date: Tue, 12 Nov 2024 16:31:13 +0800 Subject: [PATCH 28/30] =?UTF-8?q?=E4=BF=AE=E5=A4=8Dtty=E5=A4=84=E7=90=86?= =?UTF-8?q?=E4=BF=A1=E5=8F=B7=E6=97=B6=E9=94=99=E8=AF=AF=E5=9C=B0=E5=B0=86?= =?UTF-8?q?=E5=89=8D=E5=8F=B0=E8=BF=9B=E7=A8=8B=E7=BB=84=E7=BD=AE=E7=A9=BA?= =?UTF-8?q?=E7=9A=84=E9=97=AE=E9=A2=98=20&=20clone=E6=97=B6=E6=9C=AA?= =?UTF-8?q?=E6=8B=B7=E8=B4=9D=E7=88=B6=E8=BF=9B=E7=A8=8Btty=E7=9A=84?= =?UTF-8?q?=E9=97=AE=E9=A2=98=20(#1043)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 修复clone时未拷贝父进程tty的问题 * 修复tty处理信号时错误地将前台进程组置空的问题 --- kernel/src/driver/tty/tty_ldisc/ntty.rs | 4 +--- kernel/src/process/fork.rs | 5 ----- kernel/src/process/mod.rs | 9 ++++++--- 3 files changed, 7 insertions(+), 11 deletions(-) diff --git a/kernel/src/driver/tty/tty_ldisc/ntty.rs b/kernel/src/driver/tty/tty_ldisc/ntty.rs index c26b3a293..df684e9d9 100644 --- a/kernel/src/driver/tty/tty_ldisc/ntty.rs +++ b/kernel/src/driver/tty/tty_ldisc/ntty.rs @@ -785,14 +785,12 @@ impl NTtyData { signal: Signal, ) { // 先处理信号 - let mut ctrl_info = tty.core().contorl_info_irqsave(); + let ctrl_info = tty.core().contorl_info_irqsave(); let pg = ctrl_info.pgid; if let Some(pg) = pg { let _ = Syscall::kill(pg, signal as i32); } - ctrl_info.pgid = None; - if !termios.local_mode.contains(LocalMode::NOFLSH) { // 重置 self.echo_head = 0; diff --git a/kernel/src/process/fork.rs b/kernel/src/process/fork.rs index bde3b0b8e..41265d139 100644 --- a/kernel/src/process/fork.rs +++ b/kernel/src/process/fork.rs @@ -169,11 +169,6 @@ impl ProcessManager { let pcb = ProcessControlBlock::new(name, new_kstack); - // TODO: 注意！这里设置tty的操作不符合Linux的行为！（毕竟创建进程不一定要fork，也可以用clone来创建） - // 正确做法应该是在实现进程组之后去管理前台进程组。 - pcb.sig_info_mut() - .set_tty(current_pcb.sig_info_irqsave().tty()); - let mut args = KernelCloneArgs::new(); args.flags = clone_flags; args.exit_signal = Signal::SIGCHLD; diff --git a/kernel/src/process/mod.rs b/kernel/src/process/mod.rs index 1c5ce4bb4..9049864fb 100644 --- a/kernel/src/process/mod.rs +++ b/kernel/src/process/mod.rs @@ -705,16 +705,17 @@ impl ProcessControlBlock { #[inline(never)] fn do_create_pcb(name: String, kstack: KernelStack, is_idle: bool) -> Arc { - let (pid, ppid, cwd, cred) = if is_idle { + let (pid, ppid, cwd, cred, tty) = if is_idle { let cred = INIT_CRED.clone(); - (Pid(0), Pid(0), "/".to_string(), cred) + (Pid(0), Pid(0), "/".to_string(), cred, None) } else { let ppid = ProcessManager::current_pcb().pid(); let mut cred = ProcessManager::current_pcb().cred(); cred.cap_permitted = cred.cap_ambient; cred.cap_effective = cred.cap_ambient; let cwd = ProcessManager::current_pcb().basic().cwd(); - (Self::generate_pid(), ppid, cwd, cred) + let tty = ProcessManager::current_pcb().sig_info_irqsave().tty(); + (Self::generate_pid(), ppid, cwd, cred, tty) }; let basic_info = ProcessBasicInfo::new(Pid(0), ppid, Pid(0), name, cwd, None); @@ -754,6 +755,8 @@ impl ProcessControlBlock { cred: SpinLock::new(cred), }; + pcb.sig_info.write().set_tty(tty); + // 初始化系统调用栈 #[cfg(target_arch = "x86_64")] pcb.arch_info From aaf9a5f2e637896ac28e579e92ed0d71824c098f Mon Sep 17 00:00:00 2001 From: Samuka007 Date: Wed, 13 Nov 2024 17:59:03 +0800 Subject: [PATCH 29/30] turn off syscall printing, remove unused imports and make fmt --- kernel/src/arch/x86_64/syscall/nr.rs | 2 +- kernel/src/driver/net/loopback.rs | 8 ++-- kernel/src/driver/net/mod.rs | 6 +-- kernel/src/net/posix.rs | 2 +- kernel/src/net/socket/inet/common/mod.rs | 1 - kernel/src/net/socket/inet/datagram/inner.rs | 27 +++++------ kernel/src/net/socket/inet/stream/inner.rs | 49 +++++++++++++------- kernel/src/net/socket/inet/stream/mod.rs | 28 ++++++----- kernel/src/net/socket/inet/syscall.rs | 6 +-- kernel/src/net/socket/unix/stream/mod.rs | 2 +- 10 files changed, 68 insertions(+), 63 deletions(-) diff --git a/kernel/src/arch/x86_64/syscall/nr.rs b/kernel/src/arch/x86_64/syscall/nr.rs index f3c4c895c..594a76e7b 100644 --- a/kernel/src/arch/x86_64/syscall/nr.rs +++ b/kernel/src/arch/x86_64/syscall/nr.rs @@ -732,4 +732,4 @@ impl From for usize { fn from(value: SysCall) -> Self { ::to_usize(&value).unwrap() } -} \ No newline at end of file +} diff --git a/kernel/src/driver/net/loopback.rs b/kernel/src/driver/net/loopback.rs index 5b02d54a5..5c2eddfbb 100644 --- a/kernel/src/driver/net/loopback.rs +++ b/kernel/src/driver/net/loopback.rs @@ -52,7 +52,7 @@ impl phy::RxToken for LoopbackRxToken { /// 返回函数 `f` 在 `self.buffer` 上的调用结果。 fn consume(self, f: F) -> R where - F: FnOnce(&[u8]) -> R + F: FnOnce(&[u8]) -> R, { f(self.buffer.as_slice()) } @@ -277,7 +277,7 @@ impl LoopbackInterface { /// 返回一个 `Arc`，即一个指向新创建的 `LoopbackInterface` 实例的智能指针。 pub fn new(mut driver: LoopbackDriver) -> Arc { let iface_id = generate_iface_id(); - + let hardware_addr = HardwareAddress::Ethernet(smoltcp::wire::EthernetAddress([ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, ])); @@ -296,9 +296,7 @@ impl LoopbackInterface { //设置网卡地址为127.0.0.1 iface.update_ip_addrs(|ip_addrs| { - ip_addrs - .push(cidr) - .expect("Push ipCidr failed: full"); + ip_addrs.push(cidr).expect("Push ipCidr failed: full"); }); iface.routes_mut().update(|routes_map| { diff --git a/kernel/src/driver/net/mod.rs b/kernel/src/driver/net/mod.rs index 227be9208..7b26906f4 100644 --- a/kernel/src/driver/net/mod.rs +++ b/kernel/src/driver/net/mod.rs @@ -211,9 +211,9 @@ impl IfaceCommon { let (has_events, poll_at) = { ( matches!( - interface.poll(timestamp, device, &mut sockets), + interface.poll(timestamp, device, &mut sockets), smoltcp::iface::PollResult::SocketStateChanged - ), + ), loop { let poll_at = interface.poll_at(timestamp, &sockets); let Some(instant) = poll_at else { @@ -222,7 +222,7 @@ impl IfaceCommon { if instant > timestamp { break poll_at; } - } + }, ) }; diff --git a/kernel/src/net/posix.rs b/kernel/src/net/posix.rs index 681a4c485..48024211b 100644 --- a/kernel/src/net/posix.rs +++ b/kernel/src/net/posix.rs @@ -232,7 +232,7 @@ impl SockAddr { }; return Ok(Endpoint::Unixpath((inode.metadata()?.inode_id, path))); - }, + } _ => { log::warn!("not support address family {:?}", addr.family); return Err(SystemError::EINVAL); diff --git a/kernel/src/net/socket/inet/common/mod.rs b/kernel/src/net/socket/inet/common/mod.rs index 58c209c89..2fa696138 100644 --- a/kernel/src/net/socket/inet/common/mod.rs +++ b/kernel/src/net/socket/inet/common/mod.rs @@ -1,6 +1,5 @@ use crate::net::{Iface, NET_DEVICES}; use alloc::sync::Arc; -use alloc::vec::Vec; use system_error::SystemError::{self, *}; pub mod port; diff --git a/kernel/src/net/socket/inet/datagram/inner.rs b/kernel/src/net/socket/inet/datagram/inner.rs index e27e0276c..8c964a64e 100644 --- a/kernel/src/net/socket/inet/datagram/inner.rs +++ b/kernel/src/net/socket/inet/datagram/inner.rs @@ -32,10 +32,7 @@ impl UnboundUdp { return Self { socket }; } - pub fn bind( - self, - local_endpoint: smoltcp::wire::IpEndpoint, - ) -> Result { + pub fn bind(self, local_endpoint: smoltcp::wire::IpEndpoint) -> Result { // let (addr, port) = (local_endpoint.addr, local_endpoint.port); // if self.socket.bind(local_endpoint).is_err() { // log::debug!("bind failed!"); @@ -44,25 +41,25 @@ impl UnboundUdp { let inner = BoundInner::bind(self.socket, &local_endpoint.addr)?; let bind_addr = local_endpoint.addr; let bind_port = if local_endpoint.port == 0 { - inner - .port_manager() - .bind_ephemeral_port(InetTypes::Udp)? + inner.port_manager().bind_ephemeral_port(InetTypes::Udp)? } else { local_endpoint.port }; if bind_addr.is_unspecified() { - if inner.with_mut::(|socket| { - socket.bind(bind_port) - }).is_err() { + if inner + .with_mut::(|socket| socket.bind(bind_port)) + .is_err() + { return Err(SystemError::EINVAL); } - } else { - if inner.with_mut::(|socket| { + } else if inner + .with_mut::(|socket| { socket.bind(smoltcp::wire::IpEndpoint::new(bind_addr, bind_port)) - }).is_err() { - return Err(SystemError::EINVAL); - } + }) + .is_err() + { + return Err(SystemError::EINVAL); } Ok(BoundUdp { inner, diff --git a/kernel/src/net/socket/inet/stream/inner.rs b/kernel/src/net/socket/inet/stream/inner.rs index e63e2081f..36b32313e 100644 --- a/kernel/src/net/socket/inet/stream/inner.rs +++ b/kernel/src/net/socket/inet/stream/inner.rs @@ -8,8 +8,6 @@ use alloc::vec::Vec; use smoltcp; use system_error::SystemError::{self, *}; -use super::inet::UNSPECIFIED_LOCAL_ENDPOINT_V4; - // pub const DEFAULT_METADATA_BUF_SIZE: usize = 1024; pub const DEFAULT_RX_BUF_SIZE: usize = 512 * 1024; pub const DEFAULT_TX_BUF_SIZE: usize = 512 * 1024; @@ -31,7 +29,12 @@ where #[derive(Debug)] pub enum Init { - Unbound((Box>, smoltcp::wire::IpVersion)), + Unbound( + ( + Box>, + smoltcp::wire::IpVersion, + ), + ), Bound((socket::inet::BoundInner, smoltcp::wire::IpEndpoint)), } @@ -136,9 +139,12 @@ impl Init { // -1 because the first one is already bound let new_listen = socket::inet::BoundInner::bind( new_listen_smoltcp_socket(listen_addr), - listen_addr.addr.as_ref().unwrap_or( - &smoltcp::wire::IpAddress::from(smoltcp::wire::Ipv4Address::UNSPECIFIED) - ), + listen_addr + .addr + .as_ref() + .unwrap_or(&smoltcp::wire::IpAddress::from( + smoltcp::wire::Ipv4Address::UNSPECIFIED, + )), )?; inners.push(new_listen); } @@ -272,20 +278,22 @@ impl Listening { return Err(EAGAIN_OR_EWOULDBLOCK); } - let remote_endpoint = connected - .with::(|socket| { - socket - .remote_endpoint() - .expect("A Connected Tcp With No Remote Endpoint") - }); + let remote_endpoint = connected.with::(|socket| { + socket + .remote_endpoint() + .expect("A Connected Tcp With No Remote Endpoint") + }); // log::debug!("local at {:?}", local_endpoint); let mut new_listen = socket::inet::BoundInner::bind( new_listen_smoltcp_socket(self.listen_addr), - self.listen_addr.addr.as_ref().unwrap_or( - &smoltcp::wire::IpAddress::from(smoltcp::wire::Ipv4Address::UNSPECIFIED), - ), + self.listen_addr + .addr + .as_ref() + .unwrap_or(&smoltcp::wire::IpAddress::from( + smoltcp::wire::Ipv4Address::UNSPECIFIED, + )), )?; // swap the connected socket with the new_listen socket @@ -318,9 +326,14 @@ impl Listening { } pub fn get_name(&self) -> smoltcp::wire::IpEndpoint { - smoltcp::wire::IpEndpoint::new(self.listen_addr.addr.unwrap_or( - smoltcp::wire::IpAddress::from(smoltcp::wire::Ipv4Address::UNSPECIFIED), - ), self.listen_addr.port) + smoltcp::wire::IpEndpoint::new( + self.listen_addr + .addr + .unwrap_or(smoltcp::wire::IpAddress::from( + smoltcp::wire::Ipv4Address::UNSPECIFIED, + )), + self.listen_addr.port, + ) } pub fn close(&self) { diff --git a/kernel/src/net/socket/inet/stream/mod.rs b/kernel/src/net/socket/inet/stream/mod.rs index 0c2b9ff99..935a93fa4 100644 --- a/kernel/src/net/socket/inet/stream/mod.rs +++ b/kernel/src/net/socket/inet/stream/mod.rs @@ -2,7 +2,7 @@ use alloc::sync::{Arc, Weak}; use core::sync::atomic::{AtomicBool, AtomicUsize}; use system_error::SystemError::{self, *}; -use crate::{arch::init, libs::rwlock::RwLock}; +use crate::libs::rwlock::RwLock; use crate::net::event_poll::EPollEventType; use crate::net::net_core::poll_ifaces; use crate::net::socket::*; @@ -234,12 +234,10 @@ impl Socket for TcpSocket { fn get_name(&self) -> Result { match self.inner.read().as_ref().expect("Tcp Inner is None") { - Inner::Init(Init::Unbound((_, ver))) => { - Ok(Endpoint::Ip( match ver { - smoltcp::wire::IpVersion::Ipv4 => UNSPECIFIED_LOCAL_ENDPOINT_V4, - smoltcp::wire::IpVersion::Ipv6 => UNSPECIFIED_LOCAL_ENDPOINT_V6, - })) - } + Inner::Init(Init::Unbound((_, ver))) => Ok(Endpoint::Ip(match ver { + smoltcp::wire::IpVersion::Ipv4 => UNSPECIFIED_LOCAL_ENDPOINT_V4, + smoltcp::wire::IpVersion::Ipv6 => UNSPECIFIED_LOCAL_ENDPOINT_V6, + })), Inner::Init(Init::Bound((_, local))) => Ok(Endpoint::Ip(*local)), Inner::Connecting(connecting) => Ok(Endpoint::Ip(connecting.get_name())), Inner::Established(established) => Ok(Endpoint::Ip(established.local_endpoint())), @@ -330,7 +328,7 @@ impl Socket for TcpSocket { Inner::Init(init) => { init.close(); Ok(()) - }, + } }) .unwrap_or(Ok(())) } @@ -361,7 +359,7 @@ impl Socket for TcpSocket { return Err(EINVAL); } } - }, + } KeepIntvl => { if val.len() == 4 { let mut writer = self.inner.write(); @@ -370,7 +368,9 @@ impl Socket for TcpSocket { Inner::Established(established) => { let interval = u32::from_ne_bytes([val[0], val[1], val[2], val[3]]); established.with_mut(|socket| { - socket.set_keep_alive(Some(smoltcp::time::Duration::from_secs(interval as u64))); + socket.set_keep_alive(Some(smoltcp::time::Duration::from_secs( + interval as u64, + ))); }); writer.replace(Inner::Established(established)); } @@ -382,7 +382,7 @@ impl Socket for TcpSocket { } else { return Err(EINVAL); } - }, + } KeepCnt => { // if val.len() == 4 { // let mut writer = self.inner.write(); @@ -403,10 +403,8 @@ impl Socket for TcpSocket { // } else { // return Err(EINVAL); // } - }, - KeepIdle => { - - }, + } + KeepIdle => {} _ => { log::debug!("TcpSocket::set_option: not supported"); // return Err(ENOPROTOOPT); diff --git a/kernel/src/net/socket/inet/syscall.rs b/kernel/src/net/socket/inet/syscall.rs index e7a8d58bf..2cf142a0e 100644 --- a/kernel/src/net/socket/inet/syscall.rs +++ b/kernel/src/net/socket/inet/syscall.rs @@ -47,9 +47,9 @@ pub struct Inet; impl family::Family for Inet { fn socket(stype: PSOCK, protocol: u32) -> Result, SystemError> { let socket = create_inet_socket( - smoltcp::wire::IpVersion::Ipv4, - stype, - smoltcp::wire::IpProtocol::from(protocol as u8) + smoltcp::wire::IpVersion::Ipv4, + stype, + smoltcp::wire::IpProtocol::from(protocol as u8), )?; Ok(Inode::new(socket)) } diff --git a/kernel/src/net/socket/unix/stream/mod.rs b/kernel/src/net/socket/unix/stream/mod.rs index ad443036f..b9ebc9dcd 100644 --- a/kernel/src/net/socket/unix/stream/mod.rs +++ b/kernel/src/net/socket/unix/stream/mod.rs @@ -7,7 +7,7 @@ use inner::{Connected, Init, Inner, Listener}; use log::debug; use system_error::SystemError; use unix::{ - ns::abs::{remove_abs_addr, ABSHANDLE_MAP, ABS_INODE_MAP}, + ns::abs::{remove_abs_addr, ABS_INODE_MAP}, INODE_MAP, }; From e4115c9724ba2fe6d9bc6a13100c27e4ee613dfa Mon Sep 17 00:00:00 2001 From: Samuka007 Date: Wed, 13 Nov 2024 18:02:36 +0800 Subject: [PATCH 30/30] turn off syscall printings --- kernel/src/arch/x86_64/syscall/mod.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/kernel/src/arch/x86_64/syscall/mod.rs b/kernel/src/arch/x86_64/syscall/mod.rs index 054aba3b2..4cb57d085 100644 --- a/kernel/src/arch/x86_64/syscall/mod.rs +++ b/kernel/src/arch/x86_64/syscall/mod.rs @@ -113,24 +113,24 @@ pub extern "sysv64" fn syscall_handler(frame: &mut TrapFrame) { use SysCall::*; match to_print { SYS_ACCEPT | SYS_ACCEPT4 | SYS_BIND | SYS_CONNECT | SYS_SHUTDOWN | SYS_LISTEN => { - // show &= false; + show &= false; } SYS_RECVFROM | SYS_SENDTO | SYS_SENDMSG | SYS_RECVMSG => { - // show &= false; + show &= false; } SYS_SOCKET | SYS_GETSOCKNAME | SYS_GETPEERNAME | SYS_SOCKETPAIR | SYS_SETSOCKOPT | SYS_GETSOCKOPT => { - // show &= false; + show &= false; } SYS_OPEN | SYS_OPENAT | SYS_CREAT | SYS_CLOSE => { - // show &= false; + show &= false; } SYS_READ | SYS_WRITE | SYS_READV | SYS_WRITEV | SYS_PREAD64 | SYS_PWRITE64 | SYS_PREADV | SYS_PWRITEV | SYS_PREADV2 => { show &= false; } _ => { - // show &= false; + show &= false; } } if show {