commit 6d3224a4064b7c0db0800d004d6e15f50bd8fb8b from: lex0de date: Tue Sep 1 09:27:27 2026 UTC Improve Lenix stability and performance commit - e7410e29eb092d28a616a7e1bf805d39c154d987 commit + 6d3224a4064b7c0db0800d004d6e15f50bd8fb8b blob - 6f743a8441c62c158d86795fa21f5e40f32d2398 blob + 8ef5c9e8ccc856d9de0a9a6b04c16b3a2fc33323 --- AI_CONTEXT.md +++ AI_CONTEXT.md @@ -367,9 +367,10 @@ Expected top-level logs are `boot-qemu-x86_64-uefi.log |-------------------------------|----------------------------------|------------------------------------------------| | Compile, selected arch | `make ARCH= legacy` | Successful compile and link | | Runtime sysroot compile/link | `make ARCH= test-sysroot` | Freestanding hello objects and static ELF link | -| Static analysis/style | `make lint` | Tools ran on a non-empty tracked-file list | +| Static analysis/style | `make lint` | Tools ran on a non-empty source-file list | | x86_64 UEFI boot | `make run-uefi` | Required service and shell markers | | x86_64 legacy boot | `make run-legacy` | Required service and shell markers | +| x86_64 console input | `make ARCH=x86_64 test-console` | Ordered UEFI and legacy command results | | ARM64 direct boot | `make ARCH=arm64 run-arm64` | Required common boot markers | | Cross-arch smoke | `scripts/test_sanity.sh` | Init, namesvc, VFS, and ext2 markers | | Ext2 and poll regression | `scripts/test_ext2_poll.sh` | Mount, stat, read, poll, and ppoll markers | @@ -388,9 +389,14 @@ Expected top-level logs are `boot-qemu-x86_64-uefi.log though normal guests may keep running for input. Use a build or guest exit path that can terminate cleanly when running this harness. - `scripts/run_boot_tests.sh` uses fixed OVMF paths and `/tmp/boot_.log`. -- `scripts/lint.sh` gets files with `git ls-files`. This checkout currently has - `.got` but no `.git`, so a successful lint exit may have inspected no source - files. Verify logs and file counts. +- `scripts/test-console-input.sh` boots both x86_64 paths through QEMU serial + pipes. It runs `hello`, `bench_portal_pingpong`, then `hello`, and requires + two hello results with one benchmark result in that order. Its logs are + `build/test-logs/console-input-{uefi,legacy}.log`. +- `scripts/lint.sh` discovers C sources directly under `kernel/`, `servers/`, + and `user/`, excluding generated embedded-image headers. It does not depend + on Git or GoT metadata. Verify the logs under `build/test-logs/` when a tool + reports a failure or is unavailable. Never report a skipped architecture, absent tool, timeout, or empty lint file list as a pass. @@ -471,12 +477,27 @@ inventory, IOMMU data, QEMU or hardware configuration, Available guest benchmarks are: -- `bench_syscall`: 100,000 `getpid()` calls. -- `bench_fs_read`: 1,000 open/read/close cycles with configurable - `BENCH_FS_PATH`. +- `bench_syscall`: five batches of one million calls each for the null, + `getpid()`, and `getuid()` syscalls, reporting each sample and the median. +- `bench_fs_read`: creates and validates a 64 KiB `/fstest`, then performs ten + complete 4 KiB-buffered reads through VFS, ext2, blockd, and device 2. +- `bench_block_read`: five 1 MiB sequential reads from a block device in 4 KiB + requests, reporting checksums and median raw throughput. Device 2 is the + default; pass a device number as its first argument to select another. - `bench_portal_pingpong`: 10,000 64-byte round trips; documentation says its server-spawn integration is incomplete. +The 2026-08-31 x86_64 UEFI baseline and optimized measurements used QEMU TCG, +q35, 2 GiB RAM, two configured vCPUs with Lenix in single-CPU mode, release +builds, and the initrd-promoted ramdisk rootfs. Under that configuration: + +| Benchmark | Baseline | Optimized | +|---------------------------|-------------------|-------------------| +| Null syscall median | 1,280 ns/call | 1,010 ns/call | +| Null syscall throughput | 781,250 calls/sec | 990,099 calls/sec | +| VFS/ext2 read throughput | 14,188 bytes/sec | 39,056 bytes/sec | +| Raw device 2 read median | Not measured | 76,987 bytes/sec | + Do not use expectation percentages in `user/bench/README.md` as measured results. Record raw timing, iteration count, debug flags, SMP mode, QEMU flags, host or hardware, and filesystem/backend before comparing runs. blob - 8cbed6759e558db8c894ae3f26f3718a3b539ee1 blob + 33f50d03188d4506e2d3a99e52c16a7be206124b --- GNUmakefile +++ GNUmakefile @@ -65,6 +65,7 @@ USER_APPS_ELFS := $(BUILD)/user/apps/hello.elf \ $(BUILD)/user/apps/bench_syscall.elf \ $(BUILD)/user/apps/real_read.elf \ $(BUILD)/user/apps/bench_fs_read.elf \ + $(BUILD)/user/apps/bench_block_read.elf \ $(BUILD)/user/apps/bench_portal_pingpong.elf PACKAGE_MUSL ?= 1 INCLUDE_MUSL_HEADERS ?= 1 @@ -427,7 +428,7 @@ UEFI_OBJS := endif # --------------------------- Rules ---------------------------------- -.PHONY: all legacy efi esp iso run-uefi run-legacy run run-arm64 initrd clean dirs lint package-runtime test-sysroot musl +.PHONY: all legacy efi esp iso run-uefi run-legacy run run-arm64 initrd clean dirs lint package-runtime test-sysroot test-console musl musl: $(MUSL_LIBC) @@ -458,12 +459,14 @@ dirs: # ---- compile C for legacy kernel (ELF) ---- $(ARCH_DIR)/%.o: %.c | dirs @mkdir -p $(dir $@) - $(CC) $(CFLAGS) -Ikernel -Ikernel/include -c $< -o $@ + $(CC) $(CFLAGS) -MMD -MP -MF $@.d -Ikernel -Ikernel/include \ + -c $< -o $@ # ---- GAS .S for legacy ---- $(ARCH_DIR)/%.o: %.S | dirs @mkdir -p $(dir $@) - $(CC) $(ASFLAGS) -x assembler-with-cpp -c $< -o $@ + $(CC) $(ASFLAGS) -MMD -MP -MF $@.d -x assembler-with-cpp \ + -c $< -o $@ # ---- NASM .asm for legacy (x86_64 only) ---- ifeq ($(ARCH),x86_64) @@ -474,11 +477,12 @@ endif $(USER_BUILD_DIR)/%.o: %.c | dirs @mkdir -p $(dir $@) - $(CC) $(USER_CFLAGS) -c $< -o $@ + $(CC) $(USER_CFLAGS) -MMD -MP -MF $@.d -c $< -o $@ $(USER_BUILD_DIR)/%.o: %.S | dirs @mkdir -p $(dir $@) - $(CC) $(USER_ASFLAGS) -x assembler-with-cpp -c $< -o $@ + $(CC) $(USER_ASFLAGS) -MMD -MP -MF $@.d -x assembler-with-cpp \ + -c $< -o $@ $(USER_RUNTIME_ARCHIVE): $(USER_RUNTIME_OBJS) $(USER_RUNTIME_CRT0) | dirs @mkdir -p $(dir $@) @@ -486,10 +490,16 @@ $(USER_RUNTIME_ARCHIVE): $(USER_RUNTIME_OBJS) $(USER_R apps: $(USER_RUNTIME_ARCHIVE) $(MAKE) -f $(USER_APPS_MAKEFILE) BUILD_DIR=$(BUILD) ARCH=$(ARCH) \ + CC="$(CC)" LD="$(LD)" \ + USER_CFLAGS="$(filter-out -Werror,$(USER_CFLAGS))" \ + USER_LDFLAGS="$(USER_LDFLAGS)" \ RUNTIME_ARCHIVE=$(USER_RUNTIME_ARCHIVE) apps install-apps: $(USER_RUNTIME_ARCHIVE) $(MAKE) -f $(USER_APPS_MAKEFILE) BUILD_DIR=$(BUILD) ARCH=$(ARCH) \ + CC="$(CC)" LD="$(LD)" \ + USER_CFLAGS="$(filter-out -Werror,$(USER_CFLAGS))" \ + USER_LDFLAGS="$(USER_LDFLAGS)" \ RUNTIME_ARCHIVE=$(USER_RUNTIME_ARCHIVE) install package-apps: install-apps $(ROOTFS_IMG) @@ -599,7 +609,7 @@ $(ARCH_DIR)/%.uefio: %.c | dirs $(UEFI_CC) -ffreestanding -fno-builtin -fno-stack-protector -fno-omit-frame-pointer \ -Wall -Wextra -Werror -O2 \ -Ikernel -Ikernel/include -Ikernel/uefi -DUEFI_BUILD \ - -c $< -o $@ + -MMD -MP -MF $@.d -c $< -o $@ # ---- Link UEFI PE/COFF application ---- efi: $(ARCH_DIR)/lenix.efi @@ -645,6 +655,9 @@ run-legacy: iso @LOG=boot-qemu-x86_64-legacy.log; rm -f "$$LOG"; \ ./scripts/run-qemu.sh "$$LOG" +test-console: iso esp + ./scripts/test-console-input.sh + lint: ./scripts/lint.sh @@ -665,6 +678,9 @@ run-uefi: run-legacy: @echo "Legacy GRUB flow is only available for ARCH=x86_64" && exit 1 +test-console: + @echo "Console input testing requires ARCH=x86_64" && exit 1 + endif ifeq ($(ARCH),arm64) @@ -746,6 +762,8 @@ run-debug-arm64: @echo "run-debug-arm64 is only meaningful for ARCH=arm64" && exit 1 endif +-include $(addsuffix .d,$(LEGACY_OBJS) $(UEFI_OBJS) $(USER_OBJS)) + clean: rm -rf $(BUILD) rm -f kernel/user/*_image_*.h blob - a36a18a56d0e1aeed20e94dfef47b33528d74068 blob + ccd28697393ea48c3e8bf1738a75814316c69501 --- README.md +++ README.md @@ -103,6 +103,7 @@ The main make targets can also be run directly: | `make iso` | x86_64 GRUB/Multiboot2 ISO | | `make initrd` | Initial user-mode service archive | | `make build/rootfs.ext2` | ext2 root filesystem image | +| `make test-console` | UEFI and legacy serial input test | | `ARCH=arm64 make` | arm64 kernel for QEMU's `virt` machine | `ARCH` defaults to `x86_64`. Set `PACKAGE_MUSL=0` to omit musl packaging, or @@ -137,12 +138,16 @@ root: ```bash ./scripts/test_sanity.sh +make test-console make lint ``` The smoke test builds and boots the supported QEMU paths, then checks for -required boot markers. A QEMU timeout alone does not count as a pass. Set -`SANITY_TIMEOUT=` to change its default timeout. +required boot markers. `make test-console` boots both x86_64 paths and checks +that `hello`, `bench_portal_pingpong`, and a second `hello` execute in order. +Its logs are written under `build/test-logs/`. A QEMU timeout alone does not +count as a pass. Set `SANITY_TIMEOUT=` to change the smoke test's +default timeout. Focused test scripts and logs are under `scripts/` and `build/test-logs/`. blob - bc69f8c2d60c1508f5e3361e85b1d6eb3b6baad9 blob + 2c12cffcab4db40b18b214fed5fa2db5a0750f0a --- changelog.md +++ changelog.md @@ -1,7 +1,73 @@ # Changelog +# 2026-09-01 + +## Ext2 allocation corruption + +- Corrected the group descriptor table offset used by ext2 block and inode + allocation on 4 KiB filesystems. +- Corrected group-relative block bitmap indices and one-based inode bitmap + indices in allocation and free paths. Creating `/fstest` no longer + overwrites existing application inodes or data blocks. +- Verified `bench_fs_read` followed by `bench_syscall` under legacy and UEFI + QEMU. The filesystem benchmark validated 65,536 bytes, and the syscall + benchmark loaded and completed without ELF errors. + # 2026-08-31 +## Console input regression + +- Moved COM1 IRQ4 activation after x86_64 IDT and exception setup so the UART + vector is not replaced by the default interrupt handler. +- Added an atomic RX ring between IRQ4 and the console task. Hardware polling + runs with interrupts masked, keeping the console line discipline as the only + consumer without restoring the duplicate raw-stdin path. +- Made the scheduler leave `hlt` through `sched_yield()` so an interrupt can + dispatch work that became runnable while the CPU was idle. +- Corrected the console mailbox lock probe so a failed acquisition cannot + release a lock held by another task. +- Added `make test-console`, which boots UEFI and legacy QEMU paths and verifies + `hello`, `bench_portal_pingpong`, then `hello` execute once and in order. + +## Syscall, IPC, and storage performance + +- Replaced syscall fast-table initialization and the custom-syscall linear + search with one compile-time table bounded by `SYS_MAX`. Resolved existing + syscall-number collisions and added `scripts/check-syscall-abi.sh` to compare + the kernel and runtime definitions. +- Moved x86_64 syscall stack, register, return, and exec scratch state into + GS-relative per-CPU storage. BSP and AP setup now install the correct GS base, + and ARM64 exec returns restore the saved user trap frame. +- Reduced kernel service-request stack use by sharing one bounded scratch + buffer and storing the caller's response pointer in each pending slot. The + response copy and timeout cleanup are synchronized to prevent a late response + from writing through a released slot. +- Added push-capable block backend registration. Blockd sends queued requests + directly to ramdiskd and virtio-blk, validates completion tokens and payload + lengths, and retains the older fetch protocol as a fallback. +- Made VFS the single read-ahead owner, tracked logical and backend offsets + separately, and synchronized the backend before writes or uncached reads. + Disabled the overlapping ext2 read-ahead cache. +- Added `bench_block_read`, expanded `bench_syscall` to five one-million-call + batches for three syscall paths, and made `bench_fs_read` validate all 64 KiB + before timing ten complete reads. +- Added compiler dependency files for kernel, runtime, UEFI, and application + objects. Root application sub-makes now receive the selected compiler, + linker, target flags, and sysroot flags. +- Made `scripts/lint.sh` independent of Git metadata, split kernel and userland + include paths, run the syscall ABI check, and fail on analyzer or whitespace + errors instead of accepting an empty or warning-only run. +- On x86_64 UEFI QEMU TCG, null syscall median latency changed from 1,280 to + 1,010 ns, and VFS/ext2 read throughput changed from 14,188 to 39,056 + bytes/sec. The new raw block benchmark measured 76,987 bytes/sec. Both UEFI + and legacy images reached the shell after rebuilding. +- The affected ARM64 kernel and VFS objects compile with `-Werror`. The full + ARM64 userland build still stops in the existing floating-point formatter + because it is compiled with `-mgeneral-regs-only`. +- `clang-tidy` completed on the build-wired source set. The corrected lint gate + stops on existing `cppcheck` findings; its style phase also identifies + trailing whitespace in eight existing files. + ## Ext2 indirect write performance - Restored 4 KiB ext2 write batching for aligned and partial-block operations, replacing eight 512-byte service requests with one batched request. blob - cc36766f16fd1446c452367cb0332f5d3b7cc2d5 blob + 35b589ac2c2ce4fd5d5a2f38a9474b1a57a37996 --- docs/ipc.md +++ docs/ipc.md @@ -80,10 +80,13 @@ The service-to-service IPC model is built on three lay `ipc_service_handle_response()` to: - Find the pending request entry matching `(portal, token)` - Locate the corresponding client slot in the pending registry - - Copy the response data into the client slot - - Signal the client as ready + - Copy the response into the waiting caller's bounded response buffer + - Publish the ready state and wake the waiting task - Clean up the registry entry + The registry lock remains held through the response copy and ready-state + publication so a timeout cannot release the caller's buffer concurrently. + ### Request/response flow ``` @@ -143,7 +146,12 @@ This design allows: - Tokens are 32-bit and wrap around (with 0 reserved as invalid) - Pending registry has 256 slots, supporting up to 256 concurrent requests -- Each request blocks in the kernel, spinning with yield() until response arrives -- Response data is limited to 512 bytes per request +- Each client has four pending slots. A slot stores the caller's response + pointer and capacity instead of an additional 8 KiB response array. +- Requests yield briefly, then sleep in one-tick intervals until a response or + the five-second timeout. Response handling wakes the waiting task. +- Request and response payloads are limited to + `IPC_MAILBOX_MAX_PAYLOAD`, currently 8 KiB. A response is bounded by the + caller-provided capacity. - Responses include a status code allowing services to report errors - The implementation is self-contained and requires no changes to the portal layer blob - 638c087d239f59d1f07264b1713fbbe5ea782a77 blob + 4d795893a1f19bd6516e04ced0e064e6ba83fd50 --- docs/subsystems/filesystems-vfs.md +++ docs/subsystems/filesystems-vfs.md @@ -7,6 +7,8 @@ Userspace VFS maintains the mount table, path resolver - Manage mount points, resolve paths to backends, and cache vnode metadata. - Translate POSIX-style file ops into IPC to backend filesystems. - Provide backend implementations: tmpfs (memory) and ext2 (block-backed). +- Read ahead sequential regular-file data while preserving the backend file + descriptor offset across cache hits, seeks, writes, and duplicated handles. ## Key Data Structures - `struct vfs_mount`, `struct vfs_vnode` (servers/fs/vfs/vfs_internal.h). @@ -18,6 +20,18 @@ Userspace VFS maintains the mount table, path resolver - ext2 dispatch and block I/O path (servers/fs/ext2/main.c). - Syscalls package requests through kernel IPC shims (kernel/ipc/fs_service.c, vfs_service.c). +## Read Path + +VFS owns the active sequential read-ahead cache. Each unshared regular-file +handle can hold 8 KiB and tracks both the user-visible logical offset and the +backend filesystem descriptor offset. A backend seek is sent only when those +offsets differ. `lseek()`, writes, and invalid cache state clear the cache, and +plain reads resynchronize the backend cursor before bypassing read-ahead. + +Ext2's second read-ahead layer is disabled. Keeping one cache owner avoids +duplicated 8 KiB fetches and makes backend cursor movement visible to VFS. +Ext2 still batches aligned block transfers through blockd. + ## Interactions - Namesvc registers `vfs`, `fs.ext2`, `fs.tmpfs`; clients resolve via service_resolve(). - Ext2 relies on blockd/backends; VFS relies on namesvc to find backends. @@ -25,7 +39,8 @@ Userspace VFS maintains the mount table, path resolver ## Configuration - Initial mounts set in VFS (root via tmpfs/ext2 as configured). -- Vnode cache sizing and mount table limits defined in VFS constants. +- Vnode cache sizing, mount table limits, and `VFS_READAHEAD_SIZE` are defined + in `servers/fs/vfs/vfs_internal.h`. ## TODOs - Add more filesystems, write/perms enforcement, caching policies, devfs/procfs, and mount flags (ro/rw) handling. blob - 7bbba797e69957b043805cecab555da5131ce350 blob + b33614847fd8f48ea084151945069beba2787b4c --- docs/subsystems/kernel-core.md +++ docs/subsystems/kernel-core.md @@ -13,11 +13,22 @@ Boot and bring-up path for the Lenix microkernel: pars - `struct sched_task` — task state/run-queue links (kernel/sched/task.c). - `struct vm_space`, `struct vm_mapping` — address spaces/mappings (kernel/mm/aspace.c, vm.c). - `struct bootinfo` — firmware/loader metadata (kernel/boot/bootinfo.c). +- `struct x86_per_cpu_state` holds the current task, kernel stack, CPU identity, + syscall entry scratch values, and pending exec return state for each x86_64 + CPU. ## Important Code Paths - `kmain()` (kernel/kmain.c): boot entry, initializes subsystems, launches servers. - `sched_prepare_task()`, `sched_switch()` (kernel/sched/task.c): task create/switch. - Trap print/panic paths (kernel/trap/print.c, kernel/panic.c); timer tick (kernel/sched/clock.c). +- x86_64 syscall entry uses GS-relative per-CPU state instead of global scratch + variables or an APIC lookup. BSP and AP setup assign GS before the CPU can + enter user mode. +- `syscall_dispatch()` indexes a compile-time handler table bounded by + `SYS_MAX`. `scripts/check-syscall-abi.sh` verifies matching, unique syscall + numbers in the kernel and runtime headers. +- ARM64 syscall return handles exec by restoring the saved architecture trap + frame before returning to EL0. ## Interactions - Supplies mailboxes to IPC layer; calls ELF loader for embedded/initrd images. blob - d71f846faae67e1138879756de7c4df075f78d12 blob + 2384d8eedc0f49257d90024ba43217e21cbdaae4 --- docs/subsystems/storage-block.md +++ docs/subsystems/storage-block.md @@ -1,11 +1,15 @@ # Storage / Block ## Overview -User-mode block stack with a registry daemon and backends (ramdisk, virtio-blk). Kernel forwards requests and records traces; blockd brokers geometry and dispatch to backends. +User-mode block stack with a registry daemon and ramdisk or virtio-blk +backends. The kernel forwards requests and records traces. Blockd brokers +geometry, queues requests, and dispatches them to the selected backend. ## Key Responsibilities - Track block devices and geometry in blockd; expose a backend portal. - Accept registrations from ramdiskd/virtio-blk and route I/O requests. +- Push queued work to backends that register a receive portal. +- Keep the older backend fetch protocol as a compatibility fallback. - Trace recent block operations for diagnostics. ## Key Data Structures @@ -14,7 +18,15 @@ User-mode block stack with a registry daemon and backe - Block IPC request/response structs (kernel/ipc/block_service.c, block_backend_service.c). ## Important Code Paths -- Backend register/serve loop in blockd; forwards to ramdiskd/virtio-blk portals. +- Backend registration includes `IPC_BLOCK_BACKEND_REGISTER_F_PUSH` and a + receive portal. Blockd sends `IPC_BLOCK_BACKEND_OPCODE_IO_REQUEST` directly + to that portal and receives `IPC_BLOCK_BACKEND_OPCODE_IO_RESPONSE` on its own + service portal. +- Blockd validates the device, token, transfer length, and read/write payload + shape before completing the original block-service request. One request can + be active per device, with additional requests held in a bounded queue. +- A backend that does not advertise push support continues to fetch work and + acknowledge completion through the service request protocol. - Virtio-blk: PCI discovery, MMIO mapping, DMA buffers, virtqueue service (servers/block/virtio-blk/main.c). - Ramdiskd: maps loader rootfs and serves requests synchronously (servers/block/ramdiskd/main.c). - Kernel block syscalls forward to blockd/backends (kernel/ipc/block_service.c). @@ -29,4 +41,6 @@ User-mode block stack with a registry daemon and backe - QEMU drive flags and loader ramdisk determine available devices. ## TODOs -- Add flush/barriers/error propagation, more drivers, and performance tuning for high IOPS paths. +- Add flush and barrier semantics, more drivers, and batched or shared-buffer + transfers. The current protocol carries at most 4 KiB inline and schedules + every request through the kernel, blockd, and backend. blob - fd864fd5db0d1040cd41f019d66c8b64d9db877b blob + 1b4c549921bd31732d5f135c45277ddb64a0d566 --- kernel/arch/arm64/syscall.c +++ kernel/arch/arm64/syscall.c @@ -120,6 +120,21 @@ arm64_handle_sync_el0(struct arm64_syscall_frame *fram REG(frame, 5));/* x5 */ if ((ret.flags & SYSCALL_RESULT_FLAG_EXIT) != 0) syscall_exit_to_kernel(ret.value); + if ((ret.flags & SYSCALL_RESULT_FLAG_EXECVE) != 0) { + const struct arch_trap_frame *saved; + struct sched_task *task; + + task = sched_current_task(); + saved = sched_task_user_frame(task); + if (saved == NULL) + syscall_exit_to_kernel(127); + for (size_t i = 0; i < 31; i++) + frame->regs[i] = saved->x[i]; + frame->sp_el0 = saved->sp_el0; + frame->elr_el1 = saved->elr_el1; + frame->spsr_el1 = saved->spsr_el1; + return; + } REG(frame, 0) = ret.value; } void blob - 1a95b5497b901bbb68f74cc91691ff1c0eb907f5 blob + 44fef356b005fc31cf7578a526cf0a25cbcb0e3b --- kernel/arch/x86_64/ap_entry.S +++ kernel/arch/x86_64/ap_entry.S @@ -38,6 +38,7 @@ x86_ap_entry64: movq (%rbx), %rax /* stack top */ movq %rax, %rsp movq %rax, %rdi + movl 8(%rbx), %esi /* logical CPU id */ call x86_ap_stage1 movl 8(%rbx), %edi /* logical CPU id */ movl 12(%rbx), %esi /* APIC id */ blob - 1fbf107d2fa9658cbad6714514469191d22061f4 blob + 9995413b5d9fef9e0289eb0d2734f6e62d486bd8 --- kernel/arch/x86_64/init.c +++ kernel/arch/x86_64/init.c @@ -5,6 +5,8 @@ #include "arch/arch.h" #include "log/serial.h" #include "arch/tss.h" +#include "arch/x86_64/per_cpu.h" +#include "arch/x86_64/serial.h" #include "arch/x86_64/syscall.h" #include "arch/x86_64/pic.h" #include "arch/x86_64/cpufeatures.h" @@ -73,10 +75,12 @@ arch_trap_init(void) uintptr_t stack_top; __asm__ __volatile__("mov %%rsp, %0" : "=r"(stack_top)); - x86_syscall_stack_top = stack_top; + x86_per_cpu_set_kernel_stack(0, stack_top); + x86_per_cpu_gs_base_set(0); x86_protection_init(stack_top); x86_64_trap_init(); x86_exception_init(); + x86_serial_irq_enable(); x86_syscall_init(); } blob - 3886cf9661c1f94ff039003cf6984e58efe3a7a9 blob + d5ce279a2139cd645676436f09183240884e89c7 --- kernel/arch/x86_64/isr.c +++ kernel/arch/x86_64/isr.c @@ -5,6 +5,7 @@ #include "arch/x86_64/idt.h" #include "arch/tss.h" +#include "arch/x86_64/per_cpu.h" #include "arch/x86_64/pic.h" #include "log/printk.h" #include "log/serial.h" @@ -32,7 +33,6 @@ static void x86_exception_panic(const char *, struct i static void x86_log_page_fault(uint64_t); static void x86_log_gpf(uint64_t); static inline uint64_t x86_read_cr2(void); -extern uint64_t x86_syscall_last_rcx; static void x86_dump_user_stack(uint64_t rsp); static void x86_dump_kernel_stack(uint64_t rsp); @@ -200,12 +200,14 @@ __attribute__((interrupt)) static void x86_fault_page(struct interrupt_frame *frame, uint64_t error_code) { bool user_mode = (frame->cs & 0x3) == 0x3; + struct x86_per_cpu_state *per_cpu = x86_per_cpu_current(); printk("[trap] page fault\n"); trap_print_hex64(" CR2=", x86_read_cr2()); trap_print_hex64(" error_code=", error_code); x86_log_page_fault(error_code); - trap_print_hex64(" last_sysret_rcx=", x86_syscall_last_rcx); + trap_print_hex64(" last_sysret_rcx=", + per_cpu != NULL ? per_cpu->syscall_last_rcx : 0); printk(" task="); printk(sched_current_task_name()); printk("\n"); blob - 7a7753ef959a0f5e17824004d9d09f2400a4a473 blob + 9a2c19b1b1864819af6a87da8ddd5337ba3a910c --- kernel/arch/x86_64/per_cpu.c +++ kernel/arch/x86_64/per_cpu.c @@ -11,9 +11,6 @@ /* Per-CPU state array - aligned to cache line (64 bytes) */ struct x86_per_cpu_state x86_per_cpu_states[16] __attribute__((aligned(64))); -/* Current per-CPU state pointer (set via GS.base MSR) */ -static struct x86_per_cpu_state *x86_current_per_cpu_ptr = NULL; - /* * Initialize per-CPU state for a given CPU. * Called during boot for each CPU (primary + APs). @@ -33,12 +30,19 @@ x86_per_cpu_init(uint32_t cpu_id, struct sched_cpu_sta per_cpu = &x86_per_cpu_states[cpu_id]; /* Initialize per-CPU state */ + per_cpu->self = per_cpu; per_cpu->cpu_state = state; per_cpu->current_task = NULL; /* Will be set by scheduler */ per_cpu->logical_cpu_id = cpu_id; per_cpu->apic_id = 0; /* Will be set by caller */ per_cpu->kernel_stack_top = stack_top; per_cpu->tss_ptr = 0; /* Will be set by caller */ + per_cpu->syscall_user_sp = 0; + per_cpu->syscall_regs_ptr = 0; + per_cpu->syscall_saved_rdi = 0; + per_cpu->syscall_last_rcx = 0; + per_cpu->syscall_execve_entry = 0; + per_cpu->syscall_execve_stack = 0; printk("[x86] CPU "); printk_dec(cpu_id); @@ -65,14 +69,12 @@ x86_per_cpu_gs_base_set(uint32_t cpu_id) } per_cpu_addr = (uint64_t)&x86_per_cpu_states[cpu_id]; - x86_current_per_cpu_ptr = &x86_per_cpu_states[cpu_id]; /* - * Set IA32_KERNEL_GS_BASE (0xc0000102) to per-CPU state address. - * This MSR is used by SYSCALL/SYSRET and can be accessed from kernel. + * Set IA32_GS_BASE (0xc0000101) to the per-CPU state address. * We don't use SWAPGS in Lenix because kernel stays in GS throughout. */ - wrmsr(0xc0000102, per_cpu_addr); + wrmsr(0xc0000101, per_cpu_addr); printk("[x86] CPU "); printk_dec(cpu_id); @@ -87,7 +89,10 @@ x86_per_cpu_gs_base_set(uint32_t cpu_id) struct x86_per_cpu_state * x86_get_per_cpu_state(void) { - return x86_current_per_cpu_ptr; + struct x86_per_cpu_state *per_cpu; + + __asm__ __volatile__("movq %%gs:0, %0" : "=r"(per_cpu)); + return per_cpu; } /* @@ -97,10 +102,10 @@ x86_get_per_cpu_state(void) struct sched_task * x86_get_current_task(void) { - struct x86_per_cpu_state *per_cpu = x86_current_per_cpu_ptr; - if (per_cpu == NULL) - return NULL; - return per_cpu->current_task; + struct sched_task *task; + + __asm__ __volatile__("movq %%gs:16, %0" : "=r"(task)); + return task; } /* @@ -110,10 +115,10 @@ x86_get_current_task(void) uint32_t x86_get_logical_cpu_id(void) { - struct x86_per_cpu_state *per_cpu = x86_current_per_cpu_ptr; - if (per_cpu == NULL) - return 0; - return per_cpu->logical_cpu_id; + uint32_t cpu_id; + + __asm__ __volatile__("movl %%gs:24, %0" : "=r"(cpu_id)); + return cpu_id; } /* @@ -139,3 +144,11 @@ x86_per_cpu_set_cpu_state(uint32_t cpu_id, struct sche return; x86_per_cpu_states[cpu_id].cpu_state = state; } + +void +x86_per_cpu_set_kernel_stack(uint32_t cpu_id, uint64_t stack_top) +{ + if (cpu_id >= 16) + return; + x86_per_cpu_states[cpu_id].kernel_stack_top = stack_top; +} blob - 2619127e3a7c99b3535a12be99e865ab51a35afd blob + 75f7e9547f49ba6e632934c1e5ad6b43899ff786 --- kernel/arch/x86_64/per_cpu.h +++ kernel/arch/x86_64/per_cpu.h @@ -16,12 +16,29 @@ * - Scales with call frequency (10K syscalls/sec = 1.1-2.1M cycles saved) */ +#define X86_PER_CPU_OFFSET_SELF 0 +#define X86_PER_CPU_OFFSET_CPU_STATE 8 +#define X86_PER_CPU_OFFSET_CURRENT_TASK 16 +#define X86_PER_CPU_OFFSET_LOGICAL_CPU 24 +#define X86_PER_CPU_OFFSET_KERNEL_STACK 32 +#define X86_PER_CPU_OFFSET_TSS 40 +#define X86_PER_CPU_OFFSET_USER_SP 48 +#define X86_PER_CPU_OFFSET_REGS_PTR 56 +#define X86_PER_CPU_OFFSET_SAVED_RDI 64 +#define X86_PER_CPU_OFFSET_LAST_RCX 72 +#define X86_PER_CPU_OFFSET_EXECVE_ENTRY 80 +#define X86_PER_CPU_OFFSET_EXECVE_STACK 88 + +#ifndef __ASSEMBLER__ + +#include #include struct sched_task; struct sched_cpu_state; struct x86_per_cpu_state { + struct x86_per_cpu_state *self; /* Scheduler state - updated on context switch */ struct sched_cpu_state *cpu_state; /* Per-CPU scheduler state */ struct sched_task *current_task; /* Currently executing task */ @@ -34,24 +51,23 @@ struct x86_per_cpu_state { uint64_t kernel_stack_top; /* Top of kernel stack for this CPU */ uint64_t tss_ptr; /* TSS address for this CPU */ - /* IPC state - cached for fast path */ - uint64_t _reserved[5]; /* Reserved for future per-CPU buffers */ - - /* Padding to cache line boundary (64 bytes) - * Structure size: 64 bytes exactly (8+8+4+4+8+8+40 = 80, so we need -16 padding) - * Adjust based on actual sizeof(struct x86_per_cpu_state) - */ - uint64_t _padding[2]; /* Flexible padding to reach 64-byte alignment */ + /* Syscall entry scratch state. */ + uint64_t syscall_user_sp; + uint64_t syscall_regs_ptr; + uint64_t syscall_saved_rdi; + uint64_t syscall_last_rcx; + uint64_t syscall_execve_entry; + uint64_t syscall_execve_stack; + uint64_t _padding[4]; } __attribute__((aligned(64))); /* Per-CPU state array - one per CPU */ extern struct x86_per_cpu_state x86_per_cpu_states[16]; /* MAX_CPUS = 16 typical */ -/* Accessor macros for GS-relative addressing */ -#define X86_PER_CPU_OFFSET_CURRENT_TASK offsetof(struct x86_per_cpu_state, current_task) -#define X86_PER_CPU_OFFSET_CPU_STATE offsetof(struct x86_per_cpu_state, cpu_state) -#define X86_PER_CPU_OFFSET_LOGICAL_CPU offsetof(struct x86_per_cpu_state, logical_cpu_id) -#define X86_PER_CPU_OFFSET_KERNEL_STACK offsetof(struct x86_per_cpu_state, kernel_stack_top) +_Static_assert(offsetof(struct x86_per_cpu_state, self) == + X86_PER_CPU_OFFSET_SELF, "per-CPU self offset"); +_Static_assert(offsetof(struct x86_per_cpu_state, syscall_execve_stack) == + X86_PER_CPU_OFFSET_EXECVE_STACK, "per-CPU syscall offset"); /* Initialize per-CPU state for each CPU */ void x86_per_cpu_init(uint32_t cpu_id, struct sched_cpu_state *state, uint64_t stack_top); @@ -65,6 +81,7 @@ struct sched_task *x86_get_current_task(void); uint32_t x86_get_logical_cpu_id(void); void x86_per_cpu_set_current_task(uint32_t cpu_id, struct sched_task *task); void x86_per_cpu_set_cpu_state(uint32_t cpu_id, struct sched_cpu_state *state); +void x86_per_cpu_set_kernel_stack(uint32_t cpu_id, uint64_t stack_top); /* Get current CPU's per-CPU state (fast path for kernel code) */ static inline struct x86_per_cpu_state * @@ -94,3 +111,5 @@ x86_per_cpu_current_cpu_id(void) { return x86_get_logical_cpu_id(); } + +#endif blob - 2dbbeecb2b6794f1744db62b0631b3d4f748493e blob + 8e63e4daecd8c9bd87eda3792cfb96d31df708dc --- kernel/arch/x86_64/serial.c +++ kernel/arch/x86_64/serial.c @@ -2,11 +2,12 @@ /* Share the prototype definitions used by both legacy and UEFI paths. */ #include +#include "arch/arch.h" +#include "arch/x86_64/serial.h" #include "log/serial.h" #include "log/printk.h" #ifndef UEFI_BUILD #include "console/line.h" -#include "sched/task.h" #endif #if SERIAL_DEBUG_RX && !defined(UEFI_BUILD) @@ -26,6 +27,13 @@ extern void x86_serial_isr(void); #define SERIAL_WITH_IRQ 1 #define COM1_IRQ 4 #define COM1_VECTOR (32 + COM1_IRQ) +#define SERIAL_RX_BUFSZ 256 + +static char serial_rx_buf[SERIAL_RX_BUFSZ]; +static unsigned int serial_rx_head; +static unsigned int serial_rx_tail; +static unsigned int serial_irq_count; +static int serial_irq_enabled; #endif #define COM1 0x3F8 @@ -54,34 +62,36 @@ serial_outb(unsigned short port, unsigned char value) #if SERIAL_WITH_IRQ static void -serial_receive_char(char c) +serial_buffer_push(char c) { -#if SERIAL_DEBUG_RX - static int serial_rx_debug; -#endif + unsigned int head; + unsigned int next; + unsigned int tail; -#if SERIAL_DEBUG_RX && !defined(UEFI_BUILD) - if (serial_rx_debug < 8) { - const char hex[] = "0123456789abcdef"; - char buf[3]; - unsigned char uc = (unsigned char)c; - buf[0] = hex[(uc >> 4) & 0xf]; - buf[1] = hex[uc & 0xf]; - buf[2] = '\0'; - printk("[serial] RX char=0x"); - printk(buf); - if (uc >= 0x20 && uc <= 0x7e) { - char out[4] = { ' ', '(', (char)uc, ')'}; - printk(out); - } - printk("\n"); - serial_rx_debug++; - } -#endif -#ifndef UEFI_BUILD - console_line_rx(c); -#endif + head = __atomic_load_n(&serial_rx_head, __ATOMIC_RELAXED); + tail = __atomic_load_n(&serial_rx_tail, __ATOMIC_ACQUIRE); + next = (head + 1U) % SERIAL_RX_BUFSZ; + if (next == tail) + return; + serial_rx_buf[head] = c; + __atomic_store_n(&serial_rx_head, next, __ATOMIC_RELEASE); } + +static int +serial_buffer_pop(char *c) +{ + unsigned int head; + unsigned int tail; + + tail = __atomic_load_n(&serial_rx_tail, __ATOMIC_RELAXED); + head = __atomic_load_n(&serial_rx_head, __ATOMIC_ACQUIRE); + if (tail == head) + return 0; + *c = serial_rx_buf[tail]; + __atomic_store_n(&serial_rx_tail, + (tail + 1U) % SERIAL_RX_BUFSZ, __ATOMIC_RELEASE); + return 1; +} #endif /* @@ -106,19 +116,27 @@ serial_init(void) serial_outb(COM1 + 4, 0x0B); /* Enable IRQs, assert RTS/DSR. */ (void)serial_inb(COM1 + 5); /* Read LSR to acknowledge status. */ (void)serial_inb(COM1 + 0); /* Dummy read RBR to clear it. */ +#if !SERIAL_WITH_IRQ +#ifndef UEFI_BUILD + printk("[serial_init] POLLING mode (no IRQs)\n"); +#endif +#endif +} + #if SERIAL_WITH_IRQ - serial_outb(COM1 + 1, 0x01); /* Enable received-data interrupts. */ +void +x86_serial_irq_enable(void) +{ + if (serial_irq_enabled) + return; + serial_init(); x86_idt_set_gate(COM1_VECTOR, x86_serial_isr); pic_unmask_irq(COM1_IRQ); -#ifndef UEFI_BUILD + serial_outb(COM1 + 1, 0x01); /* Enable received-data interrupts. */ + serial_irq_enabled = 1; printk("[serial_init] IRQs ENABLED: COM1_IRQ=4 COM1_VECTOR=36\n"); -#endif -#else -#ifndef UEFI_BUILD - printk("[serial_init] POLLING mode (no IRQs)\n"); -#endif -#endif } +#endif /* * serial_ready - report whether the TX holding register is empty. @@ -167,55 +185,13 @@ serial_write(const char *s, size_t n) void serial_irq(void) { - static uint32_t irq_count = 0; - /* Log FIRST call unconditionally to confirm IRQ handler is invoked */ - if (irq_count == 0) { - printk("[serial_irq] FIRST CALL - IRQ handler active\n"); - } - if ((++irq_count % 10) == 0) { - printk("[serial_irq] called #"); - char buf[16]; - uint32_t val = irq_count / 10; - int idx = 0; - if (val == 0) { - buf[idx++] = '0'; - } else { - char tmp[16]; - int t = 0; - while (val > 0) { - tmp[t++] = '0' + (val % 10); - val /= 10; - } - while (t > 0) - buf[idx++] = tmp[--t]; - } - buf[idx] = '\0'; - printk(buf); - printk("*10\n"); - } + __atomic_add_fetch(&serial_irq_count, 1U, __ATOMIC_RELAXED); while ((serial_inb(COM1 + 2) & 0x01) == 0) { unsigned char lsr = serial_inb(COM1 + 5); if ((lsr & 0x01) == 0) break; char ch = (char)serial_inb(COM1 + 0); - /* Log every character received via IRQ */ - { - const char hex[] = "0123456789abcdef"; - printk("[serial_irq_rx] ch=0x"); - char buf[3]; - unsigned char uc = (unsigned char)ch; - buf[0] = hex[(uc >> 4) & 0xf]; - buf[1] = hex[uc & 0xf]; - buf[2] = '\0'; - printk(buf); - if (uc >= 0x20 && uc <= 0x7e) { - printk(" '"); - printk(&ch); - printk("'"); - } - printk("\n"); - } - serial_receive_char(ch); + serial_buffer_push(ch); } } @@ -227,6 +203,9 @@ serial_poll_rx(void) #endif unsigned char lsr; static uint32_t poll_call_count = 0; + static int serial_irq_logged; + uint64_t flags; + char ch; /* Log FIRST poll call to confirm function is being invoked */ if (poll_call_count == 0) { @@ -247,7 +226,7 @@ serial_poll_rx(void) printk(")\n"); } poll_call_count++; - + /* Log periodic polling to confirm serial_poll_rx is being called after shell prompt */ #ifdef LENIX_DEBUG if (poll_call_count % 100000 == 0 && poll_call_count > 0) { @@ -283,19 +262,30 @@ serial_poll_rx(void) } #endif - /* Poll hardware FIFO for new characters */ + /* Poll with interrupts masked so the task and ISR cannot both produce. */ + flags = arch_irq_save(); for (;;) { lsr = serial_inb(COM1 + 5); if ((lsr & 0x01) == 0) break; - char ch = (char)serial_inb(COM1 + 0); + ch = (char)serial_inb(COM1 + 0); + serial_buffer_push(ch); + } + arch_irq_restore(flags); + + if (!serial_irq_logged && + __atomic_load_n(&serial_irq_count, __ATOMIC_RELAXED) != 0) { + printk("[serial_irq] RX path active\n"); + serial_irq_logged = 1; + } + while (serial_buffer_pop(&ch)) { #if SERIAL_DEBUG_RX if (serial_poll_debug < 16) { serial_debug_log_char(ch); serial_poll_debug++; } #endif - serial_receive_char(ch); + console_line_rx(ch); } } #else blob - /dev/null blob + 83269adda3446a47513f604522f782406029e98c (mode 644) --- /dev/null +++ kernel/arch/x86_64/serial.h @@ -0,0 +1,4 @@ +/* SPDX-License-Identifier: ISC */ +#pragma once + +void x86_serial_irq_enable(void); blob - 38e7bbfc78a8bb640290e50fa70c5bd321043256 blob + 3a731852dba8a64c53e733bbe9c18442db941846 --- kernel/arch/x86_64/signal.c +++ kernel/arch/x86_64/signal.c @@ -12,9 +12,6 @@ #include "sys/syscall.h" #include "sys/wait.h" -extern uint64_t x86_syscall_user_sp; -struct x86_syscall_regs *x86_syscall_regs_ptr; - static inline uint32_t signal_pick(uint32_t mask) { @@ -45,7 +42,7 @@ x86_signal_prepare(struct x86_syscall_regs *regs, uint { struct sched_task *task = sched_current_task(); struct signal_state *state = sched_task_signal_state(task); - struct vm_space *space = sched_current_space(); + struct vm_space *space; struct signal_action act; struct sigframe frame; uint32_t pending; @@ -141,6 +138,9 @@ x86_signal_prepare(struct x86_syscall_regs *regs, uint return 0; } } + space = sched_current_space(); + if (space == NULL) + return 0; new_sp = user_sp; new_sp -= sizeof(sigreturn_stub); stub_addr = new_sp; blob - 5162a5dec83a4fac158b60b0d1a908e8c126afef blob + d9569e4e271d6e0dde1ffe0d3c467d71cd4636a1 --- kernel/arch/x86_64/signal.h +++ kernel/arch/x86_64/signal.h @@ -21,8 +21,6 @@ struct x86_syscall_regs { uint64_t r15; }; -extern struct x86_syscall_regs *x86_syscall_regs_ptr; - #include "sys/sigframe.h" uint64_t x86_signal_prepare(struct x86_syscall_regs *regs, blob - dc7c078dcc9ef9f2d54e3ed821c153b961d21cda blob + 0cf5ebe000e5ed67686d3cd5eecf2ae3019ec577 --- kernel/arch/x86_64/smp.c +++ kernel/arch/x86_64/smp.c @@ -14,6 +14,7 @@ #include "arch/x86_64/idt.h" #include "arch/x86_64/mm.h" #include "arch/x86_64/io.h" +#include "arch/x86_64/per_cpu.h" #include "boot/bootinfo.h" #include "mm/vm.h" #include "mm/vm_layout.h" @@ -901,8 +902,10 @@ x86_ipi_isr_dispatch(uint32_t reason) } void -x86_ap_stage1(uintptr_t stack_top) +x86_ap_stage1(uintptr_t stack_top, uint32_t cpu_id) { + x86_per_cpu_set_kernel_stack(cpu_id, stack_top); + x86_per_cpu_gs_base_set(cpu_id); x86_protection_init(stack_top); arch_set_kernel_stack(stack_top); } blob - 80bb3a82dbf40a694b64bb336cbd14dcebe0f8db blob + 61bc932187f01e1ad3371d9dd9b9abefbad78994 --- kernel/arch/x86_64/syscall.c +++ kernel/arch/x86_64/syscall.c @@ -21,13 +21,7 @@ #define RFLAGS_TF (1ULL << 8) extern void x86_syscall_entry(void); -extern void syscall_init_fastpath(void); -uint64_t x86_syscall_stack_top; -uint64_t x86_syscall_user_sp; -uint64_t x86_syscall_last_rcx; -uint64_t x86_syscall_saved_rdi; - static inline uint64_t rdmsr(uint32_t msr) { @@ -65,8 +59,5 @@ x86_syscall_init(void) efer |= (EFER_SCE | EFER_NXE); wrmsr(MSR_EFER, efer); - /* Initialize fastpath dispatch table for O(1) syscall lookup */ - syscall_init_fastpath(); - printk("* x86_64: SYSCALL path armed (STAR/LSTAR/SFMASK)\n"); } blob - 2deb81b102091005639779298cac2781b40427eb blob + a29e983cc24abb8737ea57bf0cea0a120a4a95a7 --- kernel/arch/x86_64/syscall.h +++ kernel/arch/x86_64/syscall.h @@ -4,6 +4,3 @@ #include void x86_syscall_init(void); - -extern uint64_t x86_syscall_stack_top; -extern uint64_t x86_syscall_user_sp; blob - abe3af5a0f656ee6192308b2f00b4c89b7a2c443 blob + e1a224e9c83bc98af95e66ff9acb1aae34c6ff17 --- kernel/arch/x86_64/syscall_entry.S +++ kernel/arch/x86_64/syscall_entry.S @@ -1,20 +1,15 @@ /* SPDX-License-Identifier: ISC */ +#include "per_cpu.h" + .text .globl x86_syscall_entry .extern syscall_dispatch .extern syscall_exit_to_kernel .extern sched_task_store_user_sp .extern sched_task_load_user_sp - .extern x86_syscall_stack_top - .extern x86_syscall_user_sp - .extern x86_syscall_last_rcx - .extern x86_syscall_saved_rdi .extern arch_trap_capture_syscall .extern x86_signal_prepare - .extern x86_syscall_regs_ptr - .extern x86_syscall_execve_entry - .extern x86_syscall_execve_stack .equ SYSCALL_FLAG_EXIT, 1 @@ -36,10 +31,10 @@ #define OFF_R15 112 x86_syscall_entry: - mov %rdi, x86_syscall_saved_rdi(%rip) + movq %rdi, %gs:X86_PER_CPU_OFFSET_SAVED_RDI mov %rsp, %rdi - mov %rdi, x86_syscall_user_sp(%rip) - mov x86_syscall_stack_top(%rip), %rsp + movq %rdi, %gs:X86_PER_CPU_OFFSET_USER_SP + movq %gs:X86_PER_CPU_OFFSET_KERNEL_STACK, %rsp test %rsp, %rsp jne .Lstack_ready .Lhalt: @@ -47,7 +42,7 @@ x86_syscall_entry: jmp .Lhalt .Lstack_ready: cld - mov x86_syscall_saved_rdi(%rip), %rdi + movq %gs:X86_PER_CPU_OFFSET_SAVED_RDI, %rdi pushq %r15 pushq %r14 pushq %r13 @@ -64,13 +59,13 @@ x86_syscall_entry: pushq %rcx pushq %rax - mov %rsp, x86_syscall_regs_ptr(%rip) + movq %rsp, %gs:X86_PER_CPU_OFFSET_REGS_PTR - mov x86_syscall_user_sp(%rip), %rdi + movq %gs:X86_PER_CPU_OFFSET_USER_SP, %rdi call sched_task_store_user_sp mov %rsp, %rdi - mov x86_syscall_user_sp(%rip), %rsi + movq %gs:X86_PER_CPU_OFFSET_USER_SP, %rsi call arch_trap_capture_syscall mov OFF_RAX(%rsp), %rdi /* syscall number */ @@ -95,31 +90,31 @@ x86_syscall_entry: /* Load new entry point and stack from global variables. * The new entry point needs to replace RCX on the stack (at OFF_RCX offset) * since RCX will be popped later and used as the return address for sysretq */ - movq x86_syscall_execve_entry(%rip), %r14 + movq %gs:X86_PER_CPU_OFFSET_EXECVE_ENTRY, %r14 movq %r14, OFF_RCX(%rsp) /* Replace RCX on stack with entry point */ - movq x86_syscall_execve_stack(%rip), %r14 - movq %r14, x86_syscall_user_sp(%rip) + movq %gs:X86_PER_CPU_OFFSET_EXECVE_STACK, %r14 + movq %r14, %gs:X86_PER_CPU_OFFSET_USER_SP /* Clear the global variables for next syscall */ - movq $0, x86_syscall_execve_entry(%rip) - movq $0, x86_syscall_execve_stack(%rip) + movq $0, %gs:X86_PER_CPU_OFFSET_EXECVE_ENTRY + movq $0, %gs:X86_PER_CPU_OFFSET_EXECVE_STACK jmp .Lafter_signal .Lno_execve: mov %rax, %r14 call sched_task_load_user_sp - mov %rax, x86_syscall_user_sp(%rip) + movq %rax, %gs:X86_PER_CPU_OFFSET_USER_SP mov %r14, %rax .Lafter_signal: mov %rax, %r15 /* save syscall return value */ lea OFF_RAX(%rsp), %rdi - mov x86_syscall_user_sp(%rip), %rsi + movq %gs:X86_PER_CPU_OFFSET_USER_SP, %rsi call x86_signal_prepare test %rax, %rax je .Lnosignal - mov %rax, x86_syscall_user_sp(%rip) + movq %rax, %gs:X86_PER_CPU_OFFSET_USER_SP .Lnosignal: mov %r15, %rax /* restore syscall return value */ @@ -140,8 +135,8 @@ x86_syscall_entry: popq %r14 popq %r15 - mov x86_syscall_user_sp(%rip), %rsp - mov %rcx, x86_syscall_last_rcx(%rip) + movq %gs:X86_PER_CPU_OFFSET_USER_SP, %rsp + movq %rcx, %gs:X86_PER_CPU_OFFSET_LAST_RCX sysretq .Lexit_kernel: blob - 36cb2f2bfc279c4258dfdb62dd5302102eac1f24 blob + 560fb3ad0da56c01445a740418e4feb1980e5f02 --- kernel/arch/x86_64/usermode.c +++ kernel/arch/x86_64/usermode.c @@ -5,6 +5,7 @@ #include "arch/arch.h" #include "arch/tss.h" +#include "arch/x86_64/per_cpu.h" #include "arch/x86_64/syscall.h" #include "arch/trap_frame.h" @@ -13,8 +14,12 @@ void arch_set_kernel_stack(uintptr_t stack_top) { + struct x86_per_cpu_state *per_cpu; + x86_set_kernel_stack(stack_top); - x86_syscall_stack_top = stack_top; + per_cpu = x86_per_cpu_current(); + if (per_cpu != NULL) + per_cpu->kernel_stack_top = stack_top; } static inline void blob - 5acc73906b7b64559273d029b872ebd0c26cff02 blob + 539ff96378239c33c338f8a2d27a449b5ac3e851 --- kernel/console/service.c +++ kernel/console/service.c @@ -920,12 +920,11 @@ console_tty_send_char(char c, uint32_t flags) * or kmem_alloc(). This prevents system-wide deadlocks. */ struct ipc_mailbox *mbox = ipc_portal_target_mailbox(console_tty_portal); if (mbox != NULL) { - /* Try to acquire lock with a quick check - if we can't get it immediately, - * the mailbox is likely contended and we should skip this send. */ - uint32_t lock_val = __atomic_exchange_n(&mbox->lock, 1U, __ATOMIC_ACQUIRE); - if (lock_val != 0) { - /* Lock was held - release our failed acquisition and skip send */ - __atomic_store_n(&mbox->lock, 0U, __ATOMIC_RELEASE); + uint32_t expected = 0; + + /* Never release a mailbox lock owned by another task. */ + if (!__atomic_compare_exchange_n(&mbox->lock, &expected, 1U, + 0, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED)) { return false; } /* Lock acquired - check depth */ blob - e0f3f450c0c4ff44732013e3789f1ecd59495f32 blob + 5e9d8b3af65d6843b7c4b763417e6fee1df15eda --- kernel/include/ipc/block_backend.h +++ kernel/include/ipc/block_backend.h @@ -20,6 +20,7 @@ #define IPC_BLOCK_BACKEND_F_WRITE (1U << 0) #define IPC_BLOCK_BACKEND_F_FLUSH (1U << 1) +#define IPC_BLOCK_BACKEND_REGISTER_F_PUSH (1U << 0) #define IPC_BLOCK_BACKEND_MAX_DATA 4096U @@ -27,7 +28,7 @@ struct ipc_block_backend_register { uint32_t opcode; uint32_t token; uint32_t flags; - uint32_t reserved; + uint32_t portal; uint32_t block_size; uint64_t block_count; }; blob - 36960d13c6879980f36c465591c29a32c2355623 blob + 8449767b4865dd0048e4092d2b075bda28c5fdd9 --- kernel/include/ipc/service_client.h +++ kernel/include/ipc/service_client.h @@ -27,7 +27,8 @@ struct ipc_service_pending { bool in_use; uint32_t token; - uint8_t response_buf[IPC_SERVICE_MAX_RESPONSE]; + uint8_t *response_buf; + size_t response_capacity; size_t response_len; bool ready; }; @@ -58,7 +59,7 @@ int ipc_service_client_init(struct ipc_service_client * 1. Allocates a token and pending slot * 2. Sends request with token via portal * 3. Yields until response is ready - * 4. Copies response to caller's buffer + * 4. Publishes the response in the caller's buffer * * spin_limit_override (iterations): optional iteration budget. If zero, a * default budget (~10M iterations with yields) is used. blob - d156644f12b502b06ce41d2b240449a57c42319e blob + c14a5305d78fb8963274f2721ea229c25bfcb4eb --- kernel/include/sync/spinlock.h +++ kernel/include/sync/spinlock.h @@ -4,6 +4,8 @@ #include #include +#include "arch/cpu.h" + /* * Simple spinlock implementation for SMP safety. * Uses atomic operations to provide mutual exclusion across CPUs. @@ -31,8 +33,8 @@ spinlock_lock(spinlock_t *lock) if (__atomic_compare_exchange_n(&lock->locked, &expected, 1, 0, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED)) return; - /* Pause hint to reduce contention */ - __asm__ __volatile__("pause" ::: "memory"); + /* Reduce contention while preserving the architecture hint. */ + arch_cpu_relax(); } } blob - c37453822f5a5f3cce53e81a9b7914a143c943c5 blob + c52c28f2fcac9762dbcf03815c0b3086b0d31cf7 --- kernel/include/sys/syscall.h +++ kernel/include/sys/syscall.h @@ -146,14 +146,14 @@ #define SYS_nanosleep 574U #define SYS_getdents 575U #define SYS_fstatat 576U -#define SYS_getcwd 590U -#define SYS_chdir 591U +#define SYS_getcwd 619U +#define SYS_chdir 620U #define SYS_fchdir 592U #define SYS_getppid 593U #define SYS_unlinkat 594U #define SYS_ftruncate 595U #define SYS_null 600U /* Null syscall for performance measurement (does nothing) */ -#define SYS_pty_get_previous 601U +#define SYS_pty_get_previous 621U #define SYS_srv_stage_start 602U #define SYS_srv_stage_commit 603U #define SYS_srv_stage_abort 604U @@ -171,6 +171,7 @@ #define SYS_getrlimit 616U #define SYS_setrlimit 617U #define SYS_getrusage 618U +#define SYS_MAX 621U #define AT_FDCWD (-100) #define AT_SYMLINK_NOFOLLOW 0x100 blob - ce9b1dad913836bd010d9af81b42f1247e9b613a blob + 1a8d6d939db2e3ae14ad098d49f950fea2cf64a1 --- kernel/ipc/service_client.c +++ kernel/ipc/service_client.c @@ -26,7 +26,6 @@ struct pending_service_request { uint32_t token; /* Request token */ struct sched_task *waiter; /* Task waiting on the response */ struct ipc_service_pending *pending_slot; /* Pointer to pending slot in client */ - volatile uint8_t prepared; /* Set when client is prepared to block */ }; static spinlock_t pending_requests_lock = SPINLOCK_INIT; @@ -76,8 +75,6 @@ pending_service_request_alloc(void) res = &pending_requests[i]; /* Mark as in_use under lock */ res->in_use = true; - __atomic_store_n((uint8_t *)&res->prepared, 0, - __ATOMIC_RELAXED); break; } } @@ -132,6 +129,16 @@ service_pending_find(struct ipc_service_client *client return NULL; } +static void +service_pending_release(struct ipc_service_pending *slot) +{ + slot->in_use = false; + slot->response_buf = NULL; + slot->response_capacity = 0; + slot->response_len = 0; + slot->ready = false; +} + int ipc_service_client_init(struct ipc_service_client *client, ipc_portal_handle_t portal) @@ -143,6 +150,8 @@ ipc_service_client_init(struct ipc_service_client *cli client->next_token = 1; for (size_t i = 0; i < IPC_SERVICE_MAX_PENDING; i++) { client->slots[i].in_use = false; + client->slots[i].response_buf = NULL; + client->slots[i].response_capacity = 0; client->slots[i].ready = false; client->slots[i].response_len = 0; } @@ -167,12 +176,14 @@ ipc_service_request_issue_timeout(struct ipc_service_c struct ipc_service_pending *slot; uint8_t tmp_req[IPC_MAILBOX_MAX_PAYLOAD]; uint64_t spins = 0; + size_t actual_resp_len; (void)spin_limit_override; /* Override no longer used; fixed threshold */ if (client == NULL || req == NULL || resp == NULL) return -1; /* Portal payloads are limited by IPC_MAILBOX_MAX_PAYLOAD; enforce the * same cap here so we can safely inline the request buffer. */ - if (req_len > sizeof(tmp_req)) + if (req_len < 2 * sizeof(uint32_t) || req_len > sizeof(tmp_req) || + resp_len > IPC_MAILBOX_MAX_PAYLOAD) return -1; /* Allocate a pending slot for this request */ @@ -180,20 +191,18 @@ ipc_service_request_issue_timeout(struct ipc_service_c if (slot == NULL) return -1; - if (resp_len > sizeof(slot->response_buf)) { - slot->in_use = false; - return -1; - } - slot->in_use = true; slot->ready = false; slot->token = service_next_token(); + slot->response_buf = resp; + slot->response_capacity = resp_len; + slot->response_len = 0; client->next_token = slot->token; /* Register this request in the global pending registry */ struct pending_service_request *preq = pending_service_request_alloc(); if (preq == NULL) { - slot->in_use = false; + service_pending_release(slot); return -1; } /* preq->in_use is already set by pending_service_request_alloc() */ @@ -211,10 +220,11 @@ ipc_service_request_issue_timeout(struct ipc_service_c /* Send request to service portal */ if (ipc_portal_send(client->server_portal, tmp_req, req_len) != 0) { - slot->in_use = false; spinlock_lock(&pending_requests_lock); preq->in_use = false; + preq->waiter = NULL; spinlock_unlock(&pending_requests_lock); + service_pending_release(slot); return -1; } @@ -241,11 +251,18 @@ ipc_service_request_issue_timeout(struct ipc_service_c /* Time-based timeout */ if (timeout_ticks > 0 && (timer_ticks() - start_ticks) > timeout_ticks) { - slot->in_use = false; spinlock_lock(&pending_requests_lock); - preq->in_use = false; - preq->waiter = NULL; + if (__atomic_load_n(&slot->ready, __ATOMIC_ACQUIRE)) { + spinlock_unlock(&pending_requests_lock); + break; + } + if (preq->in_use && preq->pending_slot == slot && + preq->token == slot->token) { + preq->in_use = false; + preq->waiter = NULL; + } spinlock_unlock(&pending_requests_lock); + service_pending_release(slot); return -1; } @@ -253,24 +270,13 @@ ipc_service_request_issue_timeout(struct ipc_service_c if (spins < 64) { sched_yield(); } else { - __atomic_store_n((uint8_t *)&preq->prepared, 1, - __ATOMIC_RELEASE); - /* Check ready after setting prepared to avoid lost wakeup */ + /* Recheck readiness before the bounded sleep. */ if (!__atomic_load_n(&slot->ready, __ATOMIC_ACQUIRE)) sched_sleep_ticks(sleep_interval); - /* Don't clear prepared here - only clear after ready=true */ } } - /* Clear prepared after successfully receiving response */ - __atomic_store_n((uint8_t *)&preq->prepared, 0, __ATOMIC_RELAXED); - /* Copy response to caller's buffer */ - if (slot->response_len > resp_len) - slot->response_len = resp_len; - service_memcpy(resp, slot->response_buf, slot->response_len); - - slot->in_use = false; - slot->ready = false; + actual_resp_len = slot->response_len; /* Release the registry entry if it still matches this request */ spinlock_lock(&pending_requests_lock); if (preq->in_use && @@ -282,7 +288,8 @@ ipc_service_request_issue_timeout(struct ipc_service_c } spinlock_unlock(&pending_requests_lock); if (out_resp_len != NULL) - *out_resp_len = slot->response_len; + *out_resp_len = actual_resp_len; + service_pending_release(slot); return 0; } @@ -299,12 +306,14 @@ ipc_service_response_ready(struct ipc_service_client * if (slot == NULL) return -1; - if (resp_len > sizeof(slot->response_buf)) - resp_len = sizeof(slot->response_buf); + if (slot->response_buf == NULL) + return -1; + if (resp_len > slot->response_capacity) + resp_len = slot->response_capacity; service_memcpy(slot->response_buf, resp, resp_len); slot->response_len = resp_len; - slot->ready = true; + __atomic_store_n(&slot->ready, true, __ATOMIC_RELEASE); return 0; } @@ -367,9 +376,13 @@ ipc_service_handle_response(ipc_portal_handle_t portal return -1; } - /* Copy response to the pending slot */ - if (resp_len > sizeof(slot->response_buf)) - resp_len = sizeof(slot->response_buf); + /* Copy the response directly into the caller-owned buffer. */ + if (slot->response_buf == NULL) { + spinlock_unlock(&pending_requests_lock); + return -1; + } + if (resp_len > slot->response_capacity) + resp_len = slot->response_capacity; service_memcpy(slot->response_buf, resp, resp_len); slot->response_len = resp_len; blob - 1d9880a4b3e03c77666511f6524bb9051fc1a194 blob + 0ffb7b4006139833bbb22886ae764d82a910c44e --- kernel/sched/task.c +++ kernel/sched/task.c @@ -1085,8 +1085,10 @@ fallback_exit: static void sched_idle(void *arg __attribute__((unused))) { - for (;;) + for (;;) { arch_idle(); + sched_yield(); + } } /* blob - 2ec743dac5e1265ad008330832b3832263c7ade6 blob + f68092340aeb09364057e1fe0b5117c9f8433f8f --- kernel/sys/policy.c +++ kernel/sys/policy.c @@ -52,12 +52,18 @@ struct policy_portal_rule { #define POLICY_MAX_SYSCALL_RULES 32 #define POLICY_MAX_PORTAL_RULES 32 -static struct policy_syscall_rule syscall_rules[POLICY_MAX_SYSCALL_RULES]; +static struct policy_syscall_rule syscall_rules[POLICY_MAX_SYSCALL_RULES] = { + { + .uid = POLICY_UID_NON_ROOT, + .syscall = SYS_execve, + .allow = false, + .in_use = true, + }, +}; static struct policy_portal_rule portal_rules[POLICY_MAX_PORTAL_RULES]; -static bool policy_rules_initialized; +static size_t syscall_rule_count = 1; +static size_t portal_rule_count; -static void policy_rules_init(void); - static void policy_log(const char *action, uint64_t detail, const struct sched_credentials *creds, bool allow) { @@ -83,60 +89,49 @@ policy_log(const char *action, uint64_t detail, const printk("\n"); } -static void -policy_rules_init(void) -{ - if (policy_rules_initialized) - return; - policy_rule_add_syscall(POLICY_UID_NON_ROOT, SYS_execve, false); - policy_rules_initialized = true; -} - bool policy_rule_add_syscall(uint32_t uid, uint64_t syscall, bool allow) { - for (size_t i = 0; i < POLICY_MAX_SYSCALL_RULES; i++) { - if (!syscall_rules[i].in_use) { - syscall_rules[i].in_use = true; - syscall_rules[i].uid = uid; - syscall_rules[i].syscall = syscall; - syscall_rules[i].allow = allow; - return true; - } - } - return false; + struct policy_syscall_rule *rule; + + if (syscall_rule_count >= POLICY_MAX_SYSCALL_RULES) + return false; + rule = &syscall_rules[syscall_rule_count++]; + rule->in_use = true; + rule->uid = uid; + rule->syscall = syscall; + rule->allow = allow; + return true; } bool policy_rule_add_portal(uint32_t uid, uint32_t label, uint32_t required_rights, bool allow) { - for (size_t i = 0; i < POLICY_MAX_PORTAL_RULES; i++) { - if (!portal_rules[i].in_use) { - portal_rules[i].in_use = true; - portal_rules[i].uid = uid; - portal_rules[i].label = label; - portal_rules[i].required_rights = required_rights; - portal_rules[i].allow = allow; - return true; - } - } - return false; + struct policy_portal_rule *rule; + + if (portal_rule_count >= POLICY_MAX_PORTAL_RULES) + return false; + rule = &portal_rules[portal_rule_count++]; + rule->in_use = true; + rule->uid = uid; + rule->label = label; + rule->required_rights = required_rights; + rule->allow = allow; + return true; } bool policy_syscall_allowed(uint64_t num, const uint64_t args[6]) { - policy_rules_init(); const struct sched_credentials *creds = - sched_task_credentials(sched_current_task()); + sched_task_credentials(sched_current_task()); + uint32_t uid = (creds != NULL) ? creds->uid : POLICY_UID_ANY; bool allow = true; + (void)args; - for (size_t i = 0; i < POLICY_MAX_SYSCALL_RULES; i++) { + for (size_t i = 0; i < syscall_rule_count; i++) { const struct policy_syscall_rule *rule = &syscall_rules[i]; - uint32_t uid = (creds != NULL) ? creds->uid : POLICY_UID_ANY; - if (!rule->in_use) - continue; if (rule->uid != POLICY_UID_ANY) { if (rule->uid == POLICY_UID_NON_ROOT) { if (uid == 0) @@ -144,7 +139,6 @@ policy_syscall_allowed(uint64_t num, const uint64_t ar } else if (rule->uid != uid) continue; } - continue; if (rule->syscall != UINT64_MAX && rule->syscall != num) continue; allow = rule->allow; @@ -158,16 +152,14 @@ bool policy_portal_send_allowed(ipc_portal_handle_t handle, uint32_t rights, uint32_t label, size_t len) { - policy_rules_init(); const struct sched_credentials *creds = - sched_task_credentials(sched_current_task()); + sched_task_credentials(sched_current_task()); + uint32_t uid = (creds != NULL) ? creds->uid : POLICY_UID_ANY; bool allow = true; + (void)len; - for (size_t i = 0; i < POLICY_MAX_PORTAL_RULES; i++) { + for (size_t i = 0; i < portal_rule_count; i++) { const struct policy_portal_rule *rule = &portal_rules[i]; - uint32_t uid = (creds != NULL) ? creds->uid : POLICY_UID_ANY; - if (!rule->in_use) - continue; if (rule->uid != POLICY_UID_ANY) { if (rule->uid == POLICY_UID_NON_ROOT) { if (uid == 0) @@ -175,7 +167,6 @@ policy_portal_send_allowed(ipc_portal_handle_t handle, } else if (rule->uid != uid) continue; } - continue; if (rule->label != POLICY_LABEL_ANY && rule->label != label) continue; if (rule->required_rights != 0 && blob - 58abc97dbe1f732713866996c62e648b8b9f265e blob + c8a9923d75a85ba10aa208acf67b85fe0e0f2cef --- kernel/sys/syscall.c +++ kernel/sys/syscall.c @@ -75,6 +75,7 @@ static void syscall_debug_serial_write_i64(int64_t v); #include "console/tty_protocol.h" #include "mm/vm_layout.h" #if defined(__x86_64__) +#include "arch/x86_64/per_cpu.h" #include "arch/x86_64/syscall.h" #include "arch/x86_64/signal.h" #include "arch/x86_64/io.h" @@ -89,16 +90,6 @@ static void syscall_debug_serial_write_i64(int64_t v); typedef struct syscall_result (*syscall_impl_t)(uint64_t, uint64_t, uint64_t, uint64_t, uint64_t, uint64_t); -struct syscall_entry { - uint64_t num; - syscall_impl_t fn; - const char *name; -}; - -/* Fastpath dispatch table for syscalls 0-63 (O(1) lookup for common POSIX syscalls) */ -#define SYSCALL_FAST_MAX 64 -static syscall_impl_t syscall_fast_table[SYSCALL_FAST_MAX]; - #define SYSCALL_IO_CHUNK IPC_FS_INLINE_DATA_MAX #define SYS_POLL_MAX_FDS 128 #define SYS_MAX_FDS 64 @@ -157,10 +148,52 @@ syscall_make_result(uint64_t value, uint64_t flags) return res; } -/* Global storage for execve entry point - used to bypass normal syscall return */ -uint64_t x86_syscall_execve_entry = 0; -uint64_t x86_syscall_execve_stack = 0; +static uint64_t +syscall_frame_entry(const struct arch_trap_frame *frame) +{ +#if defined(__x86_64__) + return frame->rip; +#elif defined(__aarch64__) + return frame->elr_el1; +#endif +} +static uint64_t +syscall_frame_stack(const struct arch_trap_frame *frame) +{ +#if defined(__x86_64__) + return frame->rsp; +#elif defined(__aarch64__) + return frame->sp_el0; +#endif +} + +static void +syscall_frame_set_return(struct arch_trap_frame *frame, uint64_t value) +{ +#if defined(__x86_64__) + frame->rax = value; +#elif defined(__aarch64__) + frame->x[0] = value; +#endif +} + +static void +syscall_frame_set_exec(struct arch_trap_frame *frame, uint64_t entry, + uint64_t stack) +{ +#if defined(__x86_64__) + frame->rip = entry; + frame->rsp = stack; + frame->cs = 0x2b; + frame->ss = 0x23; +#elif defined(__aarch64__) + frame->elr_el1 = entry; + frame->sp_el0 = stack; +#endif + syscall_frame_set_return(frame, 0); +} + #define SYSCALL_PATH_MAX IPC_VFS_MAX_PATH #define SYSCALL_PATH_COMPONENTS 64 #define SYSCALL_S_IFMT 0170000U @@ -1195,165 +1228,140 @@ sys_null_impl(uint64_t arg0 __attribute__((unused)), return syscall_make_result(0, 0); } -static const struct syscall_entry syscall_table[] = { - /* Null syscall - first for best branch prediction */ - { SYS_null, sys_null_impl, "null" }, - { SYS_read, sys_read_impl, "read" }, - { SYS_write, sys_write_impl, "write" }, - { SYS_open, sys_open_impl, "open" }, - { SYS_openat, sys_openat_impl, "openat" }, - { SYS_close, sys_close_impl, "close" }, - { SYS_stat, sys_stat_impl, "stat" }, - { SYS_lstat, sys_lstat_impl, "lstat" }, - { SYS_symlinkat, sys_symlinkat_impl, "symlinkat" }, - { SYS_readlinkat, sys_readlinkat_impl, "readlinkat" }, - { SYS_renameat, sys_renameat_impl, "renameat" }, - { SYS_mknodat, sys_mknodat_impl, "mknodat" }, - { SYS_mkfifoat, sys_mkfifoat_impl, "mkfifoat" }, - { SYS_getuid, sys_getuid_impl, "getuid" }, - { SYS_geteuid, sys_geteuid_impl, "geteuid" }, - { SYS_getgid, sys_getgid_impl, "getgid" }, - { SYS_getegid, sys_getegid_impl, "getegid" }, - { SYS_setuid, sys_setuid_impl, "setuid" }, - { SYS_setgid, sys_setgid_impl, "setgid" }, - { SYS_setsid, sys_setsid_impl, "setsid" }, - { SYS_waitid, sys_waitid_impl, "waitid" }, - { SYS_getrlimit, sys_getrlimit_impl, "getrlimit" }, - { SYS_setrlimit, sys_setrlimit_impl, "setrlimit" }, - { SYS_getrusage, sys_getrusage_impl, "getrusage" }, - { SYS_socketpair, sys_socketpair_impl, "socketpair" }, - { SYS_getcwd, sys_getcwd_impl, "getcwd" }, - { SYS_chdir, sys_chdir_impl, "chdir" }, - { SYS_fchdir, sys_fchdir_impl, "fchdir" }, - { SYS_fstat, sys_fstat_impl, "fstat" }, - { SYS_mmap, sys_nosys_impl, "mmap" }, - { SYS_mprotect, sys_nosys_impl, "mprotect" }, - { SYS_munmap, sys_nosys_impl, "munmap" }, - { SYS_brk, sys_nosys_impl, "brk" }, - { SYS_lseek, sys_lseek_impl, "lseek" }, - { SYS_dup, sys_dup_impl, "dup" }, - { SYS_dup3, sys_nosys_impl, "dup3" }, - { SYS_fcntl, sys_fcntl_impl, "fcntl" }, - { SYS_pipe, sys_pipe_impl, "pipe" }, - { SYS_pipe2, sys_nosys_impl, "pipe2" }, - { SYS_poll, sys_poll_impl, "poll" }, - { SYS_sigaction, sys_sigaction_impl, "sigaction" }, - { SYS_sigprocmask, sys_sigprocmask_impl, "sigprocmask" }, - { SYS_sigaltstack, sys_nosys_impl, "sigaltstack" }, - { SYS_sigreturn, sys_sigreturn_impl, "sigreturn" }, - { SYS_raise, sys_raise_impl, "raise" }, - { SYS_waitpid, sys_waitpid_impl, "waitpid" }, - { SYS_exit_group, sys_exit_group_impl, "exit_group" }, - { SYS_clone, sys_clone_impl, "clone" }, - { SYS_set_tid_address, sys_set_tid_address_impl, "set_tid_address" }, - { SYS_futex, sys_futex_impl, "futex" }, - { SYS_spawn, sys_spawn_impl, "spawn" }, - { SYS_console_tty_register, sys_console_tty_register_impl, - "console_tty_register" }, - { SYS_pty_feed_input, sys_pty_feed_impl, "pty_feed_input" }, - { SYS_console_ttyctl_register, sys_console_ttyctl_register_impl, - "console_ttyctl_register" }, - { SYS_fork, sys_fork_impl, "fork" }, - { SYS_execve, sys_execve_impl, "execve" }, - { SYS_portal_send, sys_portal_send_impl, "portal_send" }, - { SYS_portal_recv, sys_portal_recv_impl, "portal_recv" }, - { SYS_portal_sender, sys_portal_sender_impl, "portal_sender" }, - { SYS_portal_bootstrap, sys_portal_bootstrap_impl, "portal_bootstrap" }, - { SYS_service_portal, sys_service_portal_impl, "service_portal" }, - { SYS_service_request, sys_service_request_impl, "service_request" }, - { SYS_service_respond, sys_service_respond_impl, "service_respond" }, - { SYS_fs_register, sys_fs_register_impl, "fs_register" }, - { SYS_fs_respond, sys_fs_respond_impl, "fs_respond" }, - { SYS_console_open, sys_console_open_impl, "console_open" }, - { SYS_console_close, sys_console_close_impl, "console_close" }, - { SYS_console_client_pty, sys_console_client_pty_impl, "console_client_pty" }, - { SYS_pty_alloc, sys_pty_alloc_impl, "pty_alloc" }, - { SYS_pty_free, sys_pty_free_impl, "pty_free" }, - { SYS_pty_activate, sys_pty_activate_impl, "pty_activate" }, - { SYS_pty_query, sys_pty_query_impl, "pty_query" }, - { SYS_pty_enum, sys_pty_enum_impl, "pty_enum" }, - { SYS_pty_set_supervisor, sys_pty_supervisor_impl, "pty_set_supervisor" }, - { SYS_pty_set_termios, sys_pty_set_termios_impl, "pty_set_termios" }, - { SYS_pty_get_termios, sys_pty_get_termios_impl, "pty_get_termios" }, - { SYS_pty_send_signal, sys_pty_send_signal_impl, "pty_send_signal" }, - { SYS_pty_get_previous, sys_pty_get_previous_impl, "pty_get_previous" }, - { SYS_srv_stage_start, sys_srv_stage_start_impl, "srv_stage_start" }, - { SYS_srv_stage_commit, sys_srv_stage_commit_impl, "srv_stage_commit" }, - { SYS_srv_stage_abort, sys_srv_stage_abort_impl, "srv_stage_abort" }, - { SYS_vfs_register, sys_vfs_register_impl, "vfs_register" }, - { SYS_vfs_respond, sys_vfs_respond_impl, "vfs_respond" }, - { SYS_wait_services, sys_wait_services_impl, "wait_services" }, - { SYS_wait_block_service, sys_wait_block_service_impl, "wait_block_service" }, - { SYS_wait_device_registered, sys_wait_device_registered_impl, "wait_device_registered" }, - { SYS_portal_revoke, sys_portal_revoke_impl, "portal_revoke" }, - { SYS_mount, sys_mount_impl, "mount" }, - { SYS_umount, sys_umount_impl, "umount" }, - { SYS_block_info, sys_block_info_impl, "block_info" }, - { SYS_block_rw, sys_block_rw_impl, "block_rw" }, - { SYS_blocksvc_register, sys_blocksvc_register_impl, "blocksvc_register" }, - { SYS_blocksvc_request, sys_blocksvc_request_impl, "blocksvc_request" }, - { SYS_blocksvc_respond, sys_blocksvc_respond_impl, "blocksvc_respond" }, - { SYS_exit, sys_exit_impl, "exit" }, - { SYS_getpid, sys_getpid_impl, "getpid" }, - { SYS_hw_io_read, sys_hw_io_read_impl, "hw_io_read" }, - { SYS_hw_io_write, sys_hw_io_write_impl, "hw_io_write" }, - { SYS_hw_mmio_map, sys_hw_mmio_map_impl, "hw_mmio_map" }, - { SYS_hw_irq_subscribe, sys_hw_irq_subscribe_impl, "hw_irq_subscribe" }, - { SYS_hw_irq_poll, sys_hw_irq_poll_impl, "hw_irq_poll" }, - { SYS_pci_service_register, sys_pci_service_register_impl, - "pci_service_register" }, - { SYS_block_backend_register, sys_block_backend_register_impl, - "block_backend_register" }, - { SYS_namesvc_register, sys_namesvc_register_impl, - "namesvc_register" }, - { SYS_rootfs_info, sys_rootfs_info_impl, "rootfs_info" }, - { SYS_dma_alloc, sys_dma_alloc_impl, "dma_alloc" }, - { SYS_dma_free, sys_dma_free_impl, "dma_free" }, - { SYS_console_history, sys_console_history_impl, - "console_history" }, - { SYS_ioctl, sys_ioctl_impl, "ioctl" }, - { SYS_kill, sys_kill_impl, "kill" }, - { SYS_getpgrp, sys_getpgrp_impl, "getpgrp" }, - { SYS_setpgrp, sys_setpgrp_impl, "setpgrp" }, - { SYS_tcgetpgrp, sys_tcgetpgrp_impl, "tcgetpgrp" }, - { SYS_tcsetpgrp, sys_tcsetpgrp_impl, "tcsetpgrp" }, - { SYS_socket, sys_socket_impl, "socket" }, - { SYS_bind, sys_bind_impl, "bind" }, - { SYS_listen, sys_listen_impl, "listen" }, - { SYS_accept, sys_accept_impl, "accept" }, - { SYS_connect, sys_connect_impl, "connect" }, - { SYS_send, sys_send_impl, "send" }, - { SYS_recv, sys_recv_impl, "recv" }, - { SYS_sendto, sys_sendto_impl, "sendto" }, - { SYS_recvfrom, sys_recvfrom_impl, "recvfrom" }, - { SYS_sock_tcp_server_register, sys_sock_tcp_server_register_impl, - "sock_tcp_server_register" }, - { SYS_sock_udp_server_register, sys_sock_udp_server_register_impl, - "sock_udp_server_register" }, - { SYS_clock_gettime, sys_clock_gettime_impl, "clock_gettime" }, - { SYS_gettimeofday, sys_gettimeofday_impl, "gettimeofday" }, - { SYS_nanosleep, sys_nanosleep_impl, "nanosleep" }, - { SYS_getrandom, sys_nosys_impl, "getrandom" }, - { SYS_getdents, sys_getdents_impl, "getdents" }, - { SYS_fstatat, sys_fstatat_impl, "fstatat" }, +static syscall_impl_t const syscall_table[SYS_MAX + 1] = { + [SYS_null] = sys_null_impl, + [SYS_read] = sys_read_impl, + [SYS_write] = sys_write_impl, + [SYS_open] = sys_open_impl, + [SYS_openat] = sys_openat_impl, + [SYS_close] = sys_close_impl, + [SYS_stat] = sys_stat_impl, + [SYS_lstat] = sys_lstat_impl, + [SYS_symlinkat] = sys_symlinkat_impl, + [SYS_readlinkat] = sys_readlinkat_impl, + [SYS_renameat] = sys_renameat_impl, + [SYS_mknodat] = sys_mknodat_impl, + [SYS_mkfifoat] = sys_mkfifoat_impl, + [SYS_getuid] = sys_getuid_impl, + [SYS_geteuid] = sys_geteuid_impl, + [SYS_getgid] = sys_getgid_impl, + [SYS_getegid] = sys_getegid_impl, + [SYS_setuid] = sys_setuid_impl, + [SYS_setgid] = sys_setgid_impl, + [SYS_setsid] = sys_setsid_impl, + [SYS_waitid] = sys_waitid_impl, + [SYS_getrlimit] = sys_getrlimit_impl, + [SYS_setrlimit] = sys_setrlimit_impl, + [SYS_getrusage] = sys_getrusage_impl, + [SYS_socketpair] = sys_socketpair_impl, + [SYS_getcwd] = sys_getcwd_impl, + [SYS_chdir] = sys_chdir_impl, + [SYS_fchdir] = sys_fchdir_impl, + [SYS_fstat] = sys_fstat_impl, + [SYS_mmap] = sys_nosys_impl, + [SYS_mprotect] = sys_nosys_impl, + [SYS_munmap] = sys_nosys_impl, + [SYS_brk] = sys_nosys_impl, + [SYS_lseek] = sys_lseek_impl, + [SYS_dup] = sys_dup_impl, + [SYS_dup3] = sys_nosys_impl, + [SYS_fcntl] = sys_fcntl_impl, + [SYS_pipe] = sys_pipe_impl, + [SYS_pipe2] = sys_nosys_impl, + [SYS_poll] = sys_poll_impl, + [SYS_sigaction] = sys_sigaction_impl, + [SYS_sigprocmask] = sys_sigprocmask_impl, + [SYS_sigaltstack] = sys_nosys_impl, + [SYS_sigreturn] = sys_sigreturn_impl, + [SYS_raise] = sys_raise_impl, + [SYS_waitpid] = sys_waitpid_impl, + [SYS_exit_group] = sys_exit_group_impl, + [SYS_clone] = sys_clone_impl, + [SYS_set_tid_address] = sys_set_tid_address_impl, + [SYS_futex] = sys_futex_impl, + [SYS_spawn] = sys_spawn_impl, + [SYS_console_tty_register] = sys_console_tty_register_impl, + [SYS_pty_feed_input] = sys_pty_feed_impl, + [SYS_console_ttyctl_register] = sys_console_ttyctl_register_impl, + [SYS_fork] = sys_fork_impl, + [SYS_execve] = sys_execve_impl, + [SYS_portal_send] = sys_portal_send_impl, + [SYS_portal_recv] = sys_portal_recv_impl, + [SYS_portal_sender] = sys_portal_sender_impl, + [SYS_portal_bootstrap] = sys_portal_bootstrap_impl, + [SYS_service_portal] = sys_service_portal_impl, + [SYS_service_request] = sys_service_request_impl, + [SYS_service_respond] = sys_service_respond_impl, + [SYS_fs_register] = sys_fs_register_impl, + [SYS_fs_respond] = sys_fs_respond_impl, + [SYS_console_open] = sys_console_open_impl, + [SYS_console_close] = sys_console_close_impl, + [SYS_console_client_pty] = sys_console_client_pty_impl, + [SYS_pty_alloc] = sys_pty_alloc_impl, + [SYS_pty_free] = sys_pty_free_impl, + [SYS_pty_activate] = sys_pty_activate_impl, + [SYS_pty_query] = sys_pty_query_impl, + [SYS_pty_enum] = sys_pty_enum_impl, + [SYS_pty_set_supervisor] = sys_pty_supervisor_impl, + [SYS_pty_set_termios] = sys_pty_set_termios_impl, + [SYS_pty_get_termios] = sys_pty_get_termios_impl, + [SYS_pty_send_signal] = sys_pty_send_signal_impl, + [SYS_pty_get_previous] = sys_pty_get_previous_impl, + [SYS_srv_stage_start] = sys_srv_stage_start_impl, + [SYS_srv_stage_commit] = sys_srv_stage_commit_impl, + [SYS_srv_stage_abort] = sys_srv_stage_abort_impl, + [SYS_vfs_register] = sys_vfs_register_impl, + [SYS_vfs_respond] = sys_vfs_respond_impl, + [SYS_wait_services] = sys_wait_services_impl, + [SYS_wait_block_service] = sys_wait_block_service_impl, + [SYS_wait_device_registered] = sys_wait_device_registered_impl, + [SYS_portal_revoke] = sys_portal_revoke_impl, + [SYS_mount] = sys_mount_impl, + [SYS_umount] = sys_umount_impl, + [SYS_block_info] = sys_block_info_impl, + [SYS_block_rw] = sys_block_rw_impl, + [SYS_blocksvc_register] = sys_blocksvc_register_impl, + [SYS_blocksvc_request] = sys_blocksvc_request_impl, + [SYS_blocksvc_respond] = sys_blocksvc_respond_impl, + [SYS_exit] = sys_exit_impl, + [SYS_getpid] = sys_getpid_impl, + [SYS_hw_io_read] = sys_hw_io_read_impl, + [SYS_hw_io_write] = sys_hw_io_write_impl, + [SYS_hw_mmio_map] = sys_hw_mmio_map_impl, + [SYS_hw_irq_subscribe] = sys_hw_irq_subscribe_impl, + [SYS_hw_irq_poll] = sys_hw_irq_poll_impl, + [SYS_pci_service_register] = sys_pci_service_register_impl, + [SYS_block_backend_register] = sys_block_backend_register_impl, + [SYS_namesvc_register] = sys_namesvc_register_impl, + [SYS_rootfs_info] = sys_rootfs_info_impl, + [SYS_dma_alloc] = sys_dma_alloc_impl, + [SYS_dma_free] = sys_dma_free_impl, + [SYS_console_history] = sys_console_history_impl, + [SYS_ioctl] = sys_ioctl_impl, + [SYS_kill] = sys_kill_impl, + [SYS_getpgrp] = sys_getpgrp_impl, + [SYS_setpgrp] = sys_setpgrp_impl, + [SYS_tcgetpgrp] = sys_tcgetpgrp_impl, + [SYS_tcsetpgrp] = sys_tcsetpgrp_impl, + [SYS_socket] = sys_socket_impl, + [SYS_bind] = sys_bind_impl, + [SYS_listen] = sys_listen_impl, + [SYS_accept] = sys_accept_impl, + [SYS_connect] = sys_connect_impl, + [SYS_send] = sys_send_impl, + [SYS_recv] = sys_recv_impl, + [SYS_sendto] = sys_sendto_impl, + [SYS_recvfrom] = sys_recvfrom_impl, + [SYS_sock_tcp_server_register] = sys_sock_tcp_server_register_impl, + [SYS_sock_udp_server_register] = sys_sock_udp_server_register_impl, + [SYS_clock_gettime] = sys_clock_gettime_impl, + [SYS_gettimeofday] = sys_gettimeofday_impl, + [SYS_nanosleep] = sys_nanosleep_impl, + [SYS_getrandom] = sys_nosys_impl, + [SYS_getdents] = sys_getdents_impl, + [SYS_fstatat] = sys_fstatat_impl, }; -/* Initialize fastpath dispatch table for O(1) syscall lookup */ -void -syscall_init_fastpath(void) -{ - /* Zero-initialize fastpath table (NULL = no handler) */ - for (size_t i = 0; i < SYSCALL_FAST_MAX; i++) - syscall_fast_table[i] = NULL; - - /* Populate fastpath table with syscalls 0-63 */ - for (size_t i = 0; i < (sizeof(syscall_table) / sizeof(syscall_table[0])); i++) { - uint64_t num = syscall_table[i].num; - if (num < SYSCALL_FAST_MAX) - syscall_fast_table[num] = syscall_table[i].fn; - } -} - static struct syscall_result sys_read_impl(uint64_t fd, uint64_t buf_addr, uint64_t len, uint64_t arg3 __attribute__((unused)), @@ -3247,8 +3255,15 @@ sys_sigreturn_impl(uint64_t arg0 __attribute__((unused (uint32_t)frame.mask, NULL); sched_task_user_frame_set(task, &frame.frame); #if defined(__x86_64__) - if (x86_signal_restore(x86_syscall_regs_ptr, &frame) != 0) + struct x86_per_cpu_state *per_cpu = x86_per_cpu_current(); + struct x86_syscall_regs *regs; + + if (per_cpu == NULL) return syscall_make_result(-SYSCALL_EFAULT, 0); + regs = (struct x86_syscall_regs *)(uintptr_t) + per_cpu->syscall_regs_ptr; + if (x86_signal_restore(regs, &frame) != 0) + return syscall_make_result(-SYSCALL_EFAULT, 0); sched_task_store_user_sp(frame.frame.rsp); return syscall_make_result(frame.frame.rax, 0); #elif defined(__aarch64__) @@ -3644,7 +3659,8 @@ sys_fork_impl(uint64_t arg0 __attribute__((unused)), /* Create child task */ child = sched_spawn_user("child", child_space, - parent_frame->rip, parent_frame->rsp, 0); + syscall_frame_entry(parent_frame), + syscall_frame_stack(parent_frame), 0); if (child == NULL) { rc = -SYSCALL_ENOMEM; goto out_free_child_space; @@ -3658,7 +3674,7 @@ sys_fork_impl(uint64_t arg0 __attribute__((unused)), /* Set child's initial state to match parent's registers */ struct arch_trap_frame child_frame = *parent_frame; - child_frame.rax = 0; /* Child returns 0 from fork() */ + syscall_frame_set_return(&child_frame, 0); sched_task_user_frame_set(child, &child_frame); /* Inherit stdio portal from parent */ @@ -4092,11 +4108,7 @@ out_close: if (current_frame != NULL) { new_frame = *current_frame; /* Copy current register state */ } - new_frame.rip = prog.entry; - new_frame.rsp = child_sp; - new_frame.rax = 0; /* Return value for successful execve */ - new_frame.cs = 0x2b; /* User code segment */ - new_frame.ss = 0x23; /* User data segment */ + syscall_frame_set_exec(&new_frame, prog.entry, child_sp); sched_task_user_frame_set(task, &new_frame); /* Activate the new address space */ @@ -4105,9 +4117,19 @@ out_close: /* Store execve entry point and stack for special syscall return handling. * The syscall_entry code will use these to jump directly to the new entry * point instead of returning to the instruction after the syscall */ - x86_syscall_execve_entry = prog.entry; - x86_syscall_execve_stack = child_sp; +#if defined(__x86_64__) + { + struct x86_per_cpu_state *per_cpu = x86_per_cpu_current(); + if (per_cpu == NULL) { + rc = -SYSCALL_EFAULT; + goto out_free_file; + } + per_cpu->syscall_execve_entry = prog.entry; + per_cpu->syscall_execve_stack = child_sp; + } +#endif + rc = 0; out_free_file: @@ -4786,12 +4808,7 @@ sys_service_request_impl(uint64_t portal_handle, uint6 ipc_portal_handle_t portal = (ipc_portal_handle_t)portal_handle; size_t actual_resp_len = 0; struct ipc_service_client client; - uint8_t stack_req_buf[2048]; - uint8_t stack_resp_buf[2048]; - uint8_t *req_buf = stack_req_buf; - uint8_t *resp_buf = stack_resp_buf; - bool req_alloc = false; - bool resp_alloc = false; + uint8_t scratch[IPC_MAILBOX_MAX_PAYLOAD]; /* Allow responses large enough for VFS getmounts payloads */ if (portal == IPC_PORTAL_INVALID_HANDLE) @@ -4801,24 +4818,8 @@ sys_service_request_impl(uint64_t portal_handle, uint6 if (resp_len == 0 || resp_len > 8192) return syscall_make_result(-SYSCALL_EINVAL, 0); - if (req_len > sizeof(stack_req_buf)) { - req_buf = kmem_alloc(req_len); - if (req_buf == NULL) - return syscall_make_result(-SYSCALL_ENOMEM, 0); - req_alloc = true; - } - if (resp_len > sizeof(stack_resp_buf)) { - resp_buf = kmem_alloc(resp_len); - if (resp_buf == NULL) { - if (req_alloc) - kmem_free(req_buf); - return syscall_make_result(-SYSCALL_ENOMEM, 0); - } - resp_alloc = true; - } - /* Copy request from userspace */ - if (syscall_copy_from_user(req_buf, req_addr, req_len) != 0) { + if (syscall_copy_from_user(scratch, req_addr, req_len) != 0) { printk("[sys_service_request] copy_from_user failed portal=0x"); { const char hex[] = "0123456789abcdef"; @@ -4834,10 +4835,6 @@ sys_service_request_impl(uint64_t portal_handle, uint6 printk(" resp_len="); printk_dec((uint64_t)resp_len); printk("\n"); - if (req_alloc) - kmem_free(req_buf); - if (resp_alloc) - kmem_free(resp_buf); return syscall_make_result(-SYSCALL_EFAULT, 0); } @@ -4858,16 +4855,12 @@ sys_service_request_impl(uint64_t portal_handle, uint6 printk(" resp_len="); printk_dec((uint64_t)resp_len); printk("\n"); - if (req_alloc) - kmem_free(req_buf); - if (resp_alloc) - kmem_free(resp_buf); return syscall_make_result(-SYSCALL_EINVAL, 0); } /* Issue request and wait for response */ - if (ipc_service_request_issue(&client, req_buf, req_len, - resp_buf, resp_len, &actual_resp_len) != 0) { + if (ipc_service_request_issue(&client, scratch, req_len, + scratch, resp_len, &actual_resp_len) != 0) { /* Rate-limit error messages to prevent flooding serial output */ static uint32_t error_log_count = 0; static uint64_t last_error_log_ticks = 0; @@ -4902,10 +4895,6 @@ sys_service_request_impl(uint64_t portal_handle, uint6 } else { error_log_count++; } - if (req_alloc) - kmem_free(req_buf); - if (resp_alloc) - kmem_free(resp_buf); return syscall_make_result(-SYSCALL_EIO, 0); } @@ -4913,7 +4902,7 @@ sys_service_request_impl(uint64_t portal_handle, uint6 if (actual_resp_len > resp_len) actual_resp_len = resp_len; if (actual_resp_len > 0 && - syscall_copy_to_user(resp_addr, resp_buf, actual_resp_len) != 0) { + syscall_copy_to_user(resp_addr, scratch, actual_resp_len) != 0) { printk("[sys_service_request] copy_to_user failed portal=0x"); { const char hex[] = "0123456789abcdef"; @@ -4929,17 +4918,9 @@ sys_service_request_impl(uint64_t portal_handle, uint6 printk(" resp_len="); printk_dec((uint64_t)resp_len); printk("\n"); - if (req_alloc) - kmem_free(req_buf); - if (resp_alloc) - kmem_free(resp_buf); return syscall_make_result(-SYSCALL_EFAULT, 0); } - if (req_alloc) - kmem_free(req_buf); - if (resp_alloc) - kmem_free(resp_buf); return syscall_make_result(0, 0); } @@ -6700,39 +6681,21 @@ syscall_dispatch(uint64_t num, uint64_t arg0, uint64_t uint64_t arg3, uint64_t arg4, uint64_t arg5) { const uint64_t args[6] = { arg0, arg1, arg2, arg3, arg4, arg5 }; + syscall_impl_t fn; + #ifdef LENIX_DEBUG audit_log_syscall(num, args); #endif if (!policy_syscall_allowed(num, args)) - goto deny; + return syscall_make_result(-SYSCALL_EPERM, 0); - /* Fastpath: O(1) dispatch for syscalls 0-63 (common POSIX syscalls) */ - if (num < SYSCALL_FAST_MAX) { - syscall_impl_t fn = syscall_fast_table[num]; - if (fn != NULL) { - struct syscall_result res = fn(arg0, arg1, arg2, arg3, arg4, arg5); -#if defined(__x86_64__) - x86_syscall_user_sp = sched_task_load_user_sp(); -#endif - return res; - } - } - - /* Slowpath: linear search for syscalls >= 64 (custom Lenix syscalls) */ - for (size_t i = 0; i < (sizeof(syscall_table) / sizeof(syscall_table[0])); - i++) { - if (syscall_table[i].num != num) - continue; + fn = (num <= SYS_MAX) ? syscall_table[num] : NULL; + if (fn != NULL) { #ifdef LENIX_DEBUG if (num == SYS_console_ttyctl_register) printk("[diag] dispatch: ttyctl_register sysnum hit\n"); #endif - struct syscall_result res = - syscall_table[i].fn(arg0, arg1, arg2, arg3, arg4, arg5); -#if defined(__x86_64__) - x86_syscall_user_sp = sched_task_load_user_sp(); -#endif - return res; + return fn(arg0, arg1, arg2, arg3, arg4, arg5); } #if defined(DEBUG_ALL) @@ -6748,15 +6711,7 @@ syscall_dispatch(uint64_t num, uint64_t arg0, uint64_t } DEBUG_SYSCALL_LOG("\n"); #endif -#if defined(__x86_64__) - x86_syscall_user_sp = sched_task_load_user_sp(); -#endif return syscall_make_result(-SYSCALL_ENOSYS, 0); -deny: -#if defined(__x86_64__) - x86_syscall_user_sp = sched_task_load_user_sp(); -#endif - return syscall_make_result(-SYSCALL_EPERM, 0); } static int blob - /dev/null blob + 4033ef51d3b0b3dd3ae9688621056109cfa731b8 (mode 755) --- /dev/null +++ scripts/check-syscall-abi.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash +# Lenix - Developed by lex0de (lex0de@tuta.com) +# lenix/scripts/check-syscall-abi.sh + +set -euo pipefail + +ROOT=$(cd "$(dirname "$0")/.." && pwd) +KERNEL_HEADER="$ROOT/kernel/include/sys/syscall.h" +RUNTIME_HEADER="$ROOT/user/runtime/include/sys/syscall.h" +TMP_DIR=$(mktemp -d) + +cleanup() +{ + rm -rf "$TMP_DIR" +} + +extract_syscalls() +{ + local header=$1 + local output=$2 + + awk ' + $1 == "#define" && $2 ~ /^SYS_/ && $2 != "SYS_MAX" && + $3 ~ /^[0-9]+U$/ { + value = $3 + sub(/U$/, "", value) + print $2, value + } + ' "$header" | sort -u >"$output" +} + +check_duplicates() +{ + local input=$1 + local label=$2 + local duplicates + + duplicates=$(awk ' + { + if (owner[$2] != "") + print $2 ": " owner[$2] ", " $1 + else + owner[$2] = $1 + } + ' "$input") + if [[ -n "$duplicates" ]]; then + echo "[syscall-abi] duplicate numbers in $label:" >&2 + echo "$duplicates" >&2 + return 1 + fi +} + +trap cleanup EXIT + +extract_syscalls "$KERNEL_HEADER" "$TMP_DIR/kernel" +extract_syscalls "$RUNTIME_HEADER" "$TMP_DIR/runtime" + +check_duplicates "$TMP_DIR/kernel" "$KERNEL_HEADER" +check_duplicates "$TMP_DIR/runtime" "$RUNTIME_HEADER" + +if ! diff -u "$TMP_DIR/kernel" "$TMP_DIR/runtime"; then + echo "[syscall-abi] kernel and runtime syscall definitions differ" >&2 + exit 1 +fi + +echo "[syscall-abi] syscall definitions match and are unique" blob - 0a26e12b0dda5277790590f2b090e3ecb7fd4106 blob + 56259e824630770bb5e6fb5e1d432e000c87ab6e --- scripts/lint.sh +++ scripts/lint.sh @@ -1,88 +1,134 @@ #!/usr/bin/env bash +# Lenix - Developed by lex0de (lex0de@tuta.com) +# lenix/scripts/lint.sh + # Static analysis/style gate: clang-tidy, cppcheck (when installed), and # whitespace checks to enforce style(9) expectations. set -euo pipefail ROOT=$(cd "$(dirname "$0")/.." && pwd) -ALL_FILES=$(git -C "$ROOT" ls-files '*.c' '*.h' | grep -E '^(kernel|user)/' || true) -CFILES=$(git -C "$ROOT" ls-files '*.c' | grep -E '^(kernel|user)/' || true) -INCLUDE_FLAGS=( - -Ikernel - -Ikernel/include - -Iuser/runtime/include +cd "$ROOT" +mapfile -t ALL_FILES < <(find kernel servers user -type f \ + \( -name '*.c' -o -name '*.h' \) \ + ! -path 'kernel/user/*_image_*.h' | sort) +mapfile -t KERNEL_CFILES < <(find kernel -type f -name '*.c' | sort) +mapfile -t USER_CFILES < <(find servers user -type f -name '*.c' \ + ! -path 'user/runtime/src/ipc_service.c' | sort) +CFILES=("${KERNEL_CFILES[@]}" "${USER_CFILES[@]}") +if [[ ${#ALL_FILES[@]} -eq 0 || ${#CFILES[@]} -eq 0 ]]; then + echo "[lint] no source files found" >&2 + exit 1 +fi +KERNEL_INCLUDE_FLAGS=( + -Ikernel + -Ikernel/include ) +USER_INCLUDE_FLAGS=( + -Iuser/runtime/include + -Ikernel/include +) CLANG_TIDY_INCLUDES=( - -include stddef.h - -include stdint.h + -include stddef.h + -include stdint.h ) LOG_DIR="$ROOT/build/test-logs" mkdir -p "$LOG_DIR" CLANG_TIDY_LOG="$LOG_DIR/lint-clang-tidy.log" CPPCHECK_LOG="$LOG_DIR/lint-cppcheck.log" STYLE_LOG="$LOG_DIR/lint-style.log" +TMP_DIR=$(mktemp -d) + +cleanup() +{ + rm -rf "$TMP_DIR" +} + +trap cleanup EXIT : >"$CLANG_TIDY_LOG" : >"$CPPCHECK_LOG" : >"$STYLE_LOG" -run_clang_tidy() { - local clang_tidy=${CLANG_TIDY:-clang-tidy} - if ! command -v "$clang_tidy" >/dev/null 2>&1; then - echo "[lint] clang-tidy not found; skipping" | tee -a "$CLANG_TIDY_LOG" - return 0 - fi - echo "[lint] Running clang-tidy" | tee -a "$CLANG_TIDY_LOG" - local rc=0 - while IFS= read -r file; do - [[ -z "$file" ]] && continue - if [[ $file == kernel/arch/arm64/* ]]; then - echo "[lint] skipping clang-tidy for $file (arm64 inline asm unsupported)" >>"$CLANG_TIDY_LOG" - continue - fi - "$clang_tidy" "$ROOT/$file" --quiet -- -std=gnu11 "${CLANG_TIDY_INCLUDES[@]}" "${INCLUDE_FLAGS[@]}" >temp.tidy 2>&1 || rc=$? - if [[ $rc -ne 0 ]]; then - echo "[lint] clang-tidy issues in $file" | tee -a "$CLANG_TIDY_LOG" - cat temp.tidy >>"$CLANG_TIDY_LOG" - rm -f temp.tidy - return $rc - fi - rm -f temp.tidy - done <<<"$CFILES" - echo "[lint] clang-tidy passed" | tee -a "$CLANG_TIDY_LOG" +run_clang_tidy() +{ + local clang_tidy=${CLANG_TIDY:-clang-tidy} + local file + local rc + local -a include_flags + + if ! command -v "$clang_tidy" >/dev/null 2>&1; then + echo "[lint] clang-tidy not found; skipping" | tee -a "$CLANG_TIDY_LOG" + return 0 + fi + echo "[lint] Running clang-tidy" | tee -a "$CLANG_TIDY_LOG" + for file in "${CFILES[@]}"; do + if [[ $file == kernel/arch/arm64/* ]]; then + echo "[lint] skipping clang-tidy for $file (arm64 inline asm unsupported)" >>"$CLANG_TIDY_LOG" + continue + fi + if [[ $file == kernel/* ]]; then + include_flags=("${KERNEL_INCLUDE_FLAGS[@]}") + else + include_flags=("${USER_INCLUDE_FLAGS[@]}") + fi + rc=0 + "$clang_tidy" "$ROOT/$file" --quiet -- -std=gnu11 \ + "${CLANG_TIDY_INCLUDES[@]}" "${include_flags[@]}" \ + >"$TMP_DIR/clang-tidy" 2>&1 || rc=$? + if [[ $rc -ne 0 ]]; then + echo "[lint] clang-tidy issues in $file" | tee -a "$CLANG_TIDY_LOG" + cat "$TMP_DIR/clang-tidy" >>"$CLANG_TIDY_LOG" + return "$rc" + fi + done + echo "[lint] clang-tidy passed" | tee -a "$CLANG_TIDY_LOG" } -run_cppcheck() { - local cppcheck=${CPPCHECK:-cppcheck} - if ! command -v "$cppcheck" >/dev/null 2>&1; then - echo "[lint] cppcheck not found; skipping" | tee -a "$CPPCHECK_LOG" - return 0 - fi - echo "[lint] Running cppcheck" | tee -a "$CPPCHECK_LOG" - "$cppcheck" --enable=warning,performance,portability --std=gnu11 --inline-suppr \ - "${INCLUDE_FLAGS[@]}" --template='[{file}:{line}] {severity}: {message}' \ - $CFILES >>"$CPPCHECK_LOG" 2>&1 || { - echo "[lint] cppcheck reported issues" | tee -a "$CPPCHECK_LOG" - exit 1 - } - echo "[lint] cppcheck finished" | tee -a "$CPPCHECK_LOG" +run_cppcheck() +{ + local cppcheck=${CPPCHECK:-cppcheck} + + if ! command -v "$cppcheck" >/dev/null 2>&1; then + echo "[lint] cppcheck not found; skipping" | tee -a "$CPPCHECK_LOG" + return 0 + fi + echo "[lint] Running cppcheck" | tee -a "$CPPCHECK_LOG" + "$cppcheck" --enable=warning,performance,portability --std=gnu11 \ + --error-exitcode=1 --inline-suppr "${KERNEL_INCLUDE_FLAGS[@]}" \ + --template='[{file}:{line}] {severity}: {message}' \ + "${KERNEL_CFILES[@]}" >>"$CPPCHECK_LOG" 2>&1 || { + echo "[lint] cppcheck reported kernel issues" | tee -a "$CPPCHECK_LOG" + return 1 + } + "$cppcheck" --enable=warning,performance,portability --std=gnu11 \ + --error-exitcode=1 --inline-suppr "${USER_INCLUDE_FLAGS[@]}" \ + --template='[{file}:{line}] {severity}: {message}' \ + "${USER_CFILES[@]}" >>"$CPPCHECK_LOG" 2>&1 || { + echo "[lint] cppcheck reported userland issues" | tee -a "$CPPCHECK_LOG" + return 1 + } + echo "[lint] cppcheck finished" | tee -a "$CPPCHECK_LOG" } -run_style_checks() { - echo "[lint] Running style checks" | tee -a "$STYLE_LOG" - local fail=0 - while IFS= read -r file; do - [[ -z "$file" ]] && continue - if grep -n $'\s+$' "$ROOT/$file" >>"$STYLE_LOG"; then - echo "[lint] trailing whitespace in $file" | tee -a "$STYLE_LOG" - fail=1 - fi - done <<<"$ALL_FILES" - if [[ $fail -ne 0 ]]; then - echo "[lint] style issues detected; see $STYLE_LOG" | tee -a "$STYLE_LOG" - return 0 - fi - echo "[lint] style checks passed" | tee -a "$STYLE_LOG" +run_style_checks() +{ + local fail=0 + local file + + echo "[lint] Running style checks" | tee -a "$STYLE_LOG" + for file in "${ALL_FILES[@]}"; do + if grep -nE '[[:blank:]]+$' "$ROOT/$file" >>"$STYLE_LOG"; then + echo "[lint] trailing whitespace in $file" | tee -a "$STYLE_LOG" + fail=1 + fi + done + if [[ $fail -ne 0 ]]; then + echo "[lint] style issues detected; see $STYLE_LOG" | tee -a "$STYLE_LOG" + return 1 + fi + echo "[lint] style checks passed" | tee -a "$STYLE_LOG" } +"$ROOT/scripts/check-syscall-abi.sh" run_clang_tidy run_cppcheck run_style_checks blob - /dev/null blob + 368da96ea195c9e4f481ad06c12381c93696e49a (mode 755) --- /dev/null +++ scripts/test-console-input.sh @@ -0,0 +1,201 @@ +#!/usr/bin/env bash +# Lenix - Developed by lex0de (lex0de@tuta.com) +# lenix/scripts/test-console-input.sh + +set -euo pipefail + +ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +LOG_DIR="$ROOT/build/test-logs" +OVMF_CODE=${OVMF_CODE:-/usr/share/ovmf/x64/OVMF_CODE.4m.fd} +OVMF_VARS=${OVMF_VARS:-/usr/share/ovmf/x64/OVMF_VARS.4m.fd} +TEST_TIMEOUT=${CONSOLE_TEST_TIMEOUT:-45} +TEMP_DIR=$(mktemp -d /tmp/lenix-console-input.XXXXXX) +QEMU_PID= +READER_PID= +SERIAL_BASE= + +cleanup_case() +{ + if [[ -n "$QEMU_PID" ]] && kill -0 "$QEMU_PID" 2>/dev/null; then + kill "$QEMU_PID" 2>/dev/null || true + wait "$QEMU_PID" 2>/dev/null || true + fi + if [[ -n "$READER_PID" ]] && kill -0 "$READER_PID" 2>/dev/null; then + kill "$READER_PID" 2>/dev/null || true + wait "$READER_PID" 2>/dev/null || true + fi + exec 3>&- 2>/dev/null || true + exec 4>&- 2>/dev/null || true + QEMU_PID= + READER_PID= +} + +cleanup_all() +{ + cleanup_case + rm -f "$TEMP_DIR/legacy.in" "$TEMP_DIR/legacy.out" \ + "$TEMP_DIR/uefi.in" "$TEMP_DIR/uefi.out" \ + "$TEMP_DIR/OVMF_VARS.fd" + rmdir "$TEMP_DIR" 2>/dev/null || true +} + +fail() +{ + echo "[console-test] ERROR: $1" >&2 + exit 1 +} + +wait_for_count() +{ + local count + local expected=$3 + local log=$1 + local pattern=$2 + local steps=$((TEST_TIMEOUT * 4)) + + for ((count = 0; count < steps; count++)); do + local found + found=$(grep -aFc -- "$pattern" "$log" || true) + if ((found >= expected)); then + return 0 + fi + if ! kill -0 "$QEMU_PID" 2>/dev/null; then + return 1 + fi + sleep 0.25 + done + return 1 +} + +wait_for_pattern() +{ + local count + local log=$1 + local pattern=$2 + local steps=$((TEST_TIMEOUT * 4)) + + for ((count = 0; count < steps; count++)); do + if grep -aFq -- "$pattern" "$log"; then + return 0 + fi + if ! kill -0 "$QEMU_PID" 2>/dev/null; then + return 1 + fi + sleep 0.25 + done + return 1 +} + +validate_log() +{ + local bench_count + local bench_line + local label=$1 + local log=$2 + local -a hello_lines + + mapfile -t hello_lines < <(grep -anF \ + "hello from Lenix userland" "$log" || true) + bench_count=$(grep -aFc \ + "[bench_portal_pingpong] Starting portal round-trip benchmark" \ + "$log" || true) + bench_line=$(grep -anF \ + "[bench_portal_pingpong] Starting portal round-trip benchmark" \ + "$log" | head -n 1 | cut -d: -f1) + + [[ ${#hello_lines[@]} -eq 2 ]] || + fail "$label produced ${#hello_lines[@]} hello results, expected 2" + [[ $bench_count -eq 1 ]] || + fail "$label ran bench_portal_pingpong $bench_count times, expected 1" + [[ ${hello_lines[0]%%:*} -lt $bench_line ]] || + fail "$label ran the benchmark before the first hello" + [[ $bench_line -lt ${hello_lines[1]%%:*} ]] || + fail "$label ran the second hello before the benchmark" + if grep -aEiq '\[trap\].*fault|panic' "$log"; then + fail "$label reported a kernel fault or panic" + fi +} + +run_case() +{ + local label=$1 + local log="$LOG_DIR/console-input-$label.log" + local qemu_log="$LOG_DIR/console-input-$label.qemu.log" + local -a qemu_args + + SERIAL_BASE="$TEMP_DIR/$label" + rm -f "$SERIAL_BASE.in" "$SERIAL_BASE.out" "$log" "$qemu_log" + mkfifo "$SERIAL_BASE.in" "$SERIAL_BASE.out" + exec 3<>"$SERIAL_BASE.in" + exec 4<>"$SERIAL_BASE.out" + tee "$log" <&4 >/dev/null & + READER_PID=$! + + if [[ $label == uefi ]]; then + cp "$OVMF_VARS" "$TEMP_DIR/OVMF_VARS.fd" + qemu_args=( + -machine q35 + -m 2048M + -smp 2 + -drive "if=pflash,format=raw,readonly=on,file=$OVMF_CODE" + -drive "if=pflash,format=raw,file=$TEMP_DIR/OVMF_VARS.fd" + -drive "file=fat:rw:$ROOT/build,format=raw" + ) + else + qemu_args=( + -cdrom "$ROOT/build/x86_64/lenix.iso" + -m 2048M + -smp 2 + ) + fi + qemu_args+=( + -drive "if=none,id=virtio-rootfs,file=$ROOT/build/rootfs.ext2,format=raw" + -device "virtio-blk-pci,drive=virtio-rootfs" + -serial "pipe:$SERIAL_BASE" + -monitor none + -display none + -no-reboot + -no-shutdown + ) + + echo "[console-test] Booting $label" + TMPDIR="$ROOT/build/tmp" qemu-system-x86_64 "${qemu_args[@]}" \ + >"$qemu_log" 2>&1 & + QEMU_PID=$! + + wait_for_pattern "$log" "[sh] entering main loop" || + fail "$label did not reach the shell; see $log and $qemu_log" + printf 'hello\r' >&3 + wait_for_count "$log" "hello from Lenix userland" 1 || + fail "$label did not execute the first hello; see $log" + printf 'bench_portal_pingpong\r' >&3 + wait_for_pattern "$log" "[bench_portal_pingpong] Per-RTT:" || + fail "$label did not execute bench_portal_pingpong; see $log" + printf 'hello\r' >&3 + wait_for_count "$log" "hello from Lenix userland" 2 || + fail "$label did not execute the second hello; see $log" + + validate_log "$label" "$log" + cleanup_case + echo "[console-test] $label passed" +} + +trap cleanup_all EXIT +trap 'exit 130' INT +trap 'exit 143' TERM + +command -v qemu-system-x86_64 >/dev/null 2>&1 || + fail "qemu-system-x86_64 is not installed" +[[ -f "$OVMF_CODE" ]] || fail "OVMF code image not found: $OVMF_CODE" +[[ -f "$OVMF_VARS" ]] || fail "OVMF variables image not found: $OVMF_VARS" +[[ -f "$ROOT/build/x86_64/lenix.iso" ]] || fail "legacy ISO is missing" +[[ -f "$ROOT/build/EFI/BOOT/BOOTX64.EFI" ]] || fail "UEFI image is missing" +[[ -f "$ROOT/build/rootfs.ext2" ]] || fail "rootfs image is missing" + +mkdir -p "$LOG_DIR" "$ROOT/build/tmp" +cd "$ROOT" + +run_case uefi +run_case legacy + +echo "[console-test] UEFI and legacy input tests passed" blob - 46fe4c3c6a2aa7d8cb694842a67c89ae09aadfce blob + 58f20ae8c1b17ddd18da49f62d8b11496e5a41e5 --- servers/block/blockd/main.c +++ servers/block/blockd/main.c @@ -19,8 +19,11 @@ struct blockd_backend_device { uint32_t device_id; uint32_t block_size; uint64_t block_count; + portal_handle_t backend_portal; + bool push_enabled; bool request_pending; bool waiting_completion; + uint32_t completion_token; uint64_t waiting_since_ms; /* timestamp when waiting started */ struct ipc_block_service_request pending_req; struct ipc_block_service_request queue[8]; @@ -87,6 +90,9 @@ static bool blockd_backend_queue_push(struct blockd_ba static bool blockd_backend_queue_pop(struct blockd_backend_device *slot, struct ipc_block_service_request *req_out); static void blockd_backend_promote_queued(struct blockd_backend_device *slot); +static bool blockd_backend_push_request(struct blockd_backend_device *slot); +static void blockd_backend_fail_requests(struct blockd_backend_device *slot, + int32_t status); static void blockd_check_device_timeouts(void); static uint64_t blockd_now_ms(void); @@ -199,8 +205,12 @@ blockd_backend_init(void) blockd_next_backend_id = 2; for (size_t i = 0; i < BLOCKD_BACKEND_MAX_DEVICES; i++) { blockd_backend_devices[i].in_use = false; + blockd_backend_devices[i].backend_portal = + IPC_PORTAL_INVALID_HANDLE; + blockd_backend_devices[i].push_enabled = false; blockd_backend_devices[i].request_pending = false; blockd_backend_devices[i].waiting_completion = false; + blockd_backend_devices[i].completion_token = 0; blockd_backend_queue_reset(&blockd_backend_devices[i]); } } @@ -279,7 +289,68 @@ blockd_backend_promote_queued(struct blockd_backend_de slot->request_pending = true; } +static void +blockd_backend_fail_requests(struct blockd_backend_device *slot, + int32_t status) +{ + struct ipc_block_service_request req; + struct ipc_block_service_response resp; + + if (slot == NULL) + return; + if (slot->request_pending || slot->waiting_completion) { + memset(&resp, 0, sizeof(resp)); + resp.token = slot->pending_req.token; + resp.status = status; + block_service_respond(&resp); + } + while (blockd_backend_queue_pop(slot, &req)) { + memset(&resp, 0, sizeof(resp)); + resp.token = req.token; + resp.status = status; + block_service_respond(&resp); + } + slot->request_pending = false; + slot->waiting_completion = false; + slot->completion_token = 0; +} + static bool +blockd_backend_push_request(struct blockd_backend_device *slot) +{ + struct ipc_block_backend_io_request work; + const struct ipc_block_service_request *pending; + + if (slot == NULL || !slot->push_enabled || !slot->request_pending || + slot->waiting_completion) + return true; + if (slot->backend_portal == IPC_PORTAL_INVALID_HANDLE) + return false; + pending = &slot->pending_req; + memset(&work, 0, sizeof(work)); + work.opcode = IPC_BLOCK_BACKEND_OPCODE_IO_REQUEST; + work.token = pending->token; + work.status = IPC_BLOCK_BACKEND_STATUS_OK; + work.device_id = slot->device_id; + work.flags = pending->flags; + work.lba = pending->lba; + work.blocks = pending->blocks; + work.data_len = pending->data_len; + if (work.data_len > sizeof(work.data)) + work.data_len = sizeof(work.data); + if ((pending->flags & IPC_BLOCK_SERVICE_F_WRITE) != 0 && + work.data_len > 0) + memcpy(work.data, pending->data, work.data_len); + if (portal_send(slot->backend_portal, &work, sizeof(work)) != 0) + return false; + slot->request_pending = false; + slot->waiting_completion = true; + slot->completion_token = pending->token; + slot->waiting_since_ms = blockd_now_ms(); + return true; +} + +static bool blockd_backend_queue_request(struct blockd_backend_device *slot, const struct ipc_block_service_request *req) { @@ -293,6 +364,11 @@ blockd_backend_queue_request(struct blockd_backend_dev req->device, req->token, (unsigned long long)req->lba, req->blocks, req->flags); #endif + if (!blockd_backend_push_request(slot)) { + blockd_backend_fail_requests(slot, + IPC_BLOCK_BACKEND_STATUS_ERROR); + slot->in_use = false; + } return true; } if (blockd_backend_queue_push(slot, req)) { @@ -327,14 +403,24 @@ blockd_backend_handle_register(const struct ipc_block_ blockd_log("[blockd] backend register request"); blockd_log_hex64("[blockd] block_count=0x", req->block_count); blockd_log_hex64("[blockd] block_size=0x", req->block_size); + if (req->block_size == 0 || req->block_count == 0 || + ((req->flags & IPC_BLOCK_BACKEND_REGISTER_F_PUSH) != 0 && + req->portal == IPC_PORTAL_INVALID_HANDLE)) { + resp.status = IPC_BLOCK_BACKEND_STATUS_INVALID; + goto respond; + } slot = blockd_backend_alloc_slot(); if (slot == NULL) { resp.status = IPC_BLOCK_BACKEND_STATUS_NO_DEVICE; } else { slot->in_use = true; slot->device_id = blockd_next_backend_id++; + slot->backend_portal = (portal_handle_t)req->portal; + slot->push_enabled = + (req->flags & IPC_BLOCK_BACKEND_REGISTER_F_PUSH) != 0; slot->request_pending = false; slot->waiting_completion = false; + slot->completion_token = 0; blockd_backend_queue_reset(slot); slot->block_size = req->block_size; slot->block_count = req->block_count; @@ -356,6 +442,7 @@ blockd_backend_handle_register(const struct ipc_block_ register_log_count++; } } +respond: if (blockd_backend_portal != IPC_PORTAL_INVALID_HANDLE) ipc_service_respond(blockd_backend_portal, resp.token, &resp, sizeof(resp)); @@ -375,9 +462,11 @@ blockd_backend_handle_unregister(const struct ipc_bloc resp.status = IPC_BLOCK_BACKEND_STATUS_ERROR; slot = blockd_backend_find(req->device_id); if (slot != NULL) { + blockd_backend_fail_requests(slot, + IPC_BLOCK_BACKEND_STATUS_NO_DEVICE); slot->in_use = false; - slot->request_pending = false; - slot->waiting_completion = false; + slot->backend_portal = IPC_PORTAL_INVALID_HANDLE; + slot->push_enabled = false; blockd_backend_queue_reset(slot); resp.status = IPC_BLOCK_BACKEND_STATUS_OK; } @@ -453,6 +542,7 @@ blockd_backend_handle_io_fetch(const struct ipc_block_ memcpy(resp.data, pending->data, resp.data_len); slot->request_pending = false; slot->waiting_completion = true; + slot->completion_token = resp.token; slot->waiting_since_ms = blockd_now_ms(); #ifdef LENIX_DEBUG blockd_log_int("[blockd] backend fetch token=", @@ -486,6 +576,8 @@ blockd_backend_handle_io_response( { struct blockd_backend_device *slot; struct ipc_block_service_response out; + uint64_t expected; + bool write; if (resp_pkt == NULL) return; @@ -494,6 +586,11 @@ blockd_backend_handle_io_response( return; if (!slot->waiting_completion) return; + if (resp_pkt->token != slot->completion_token) { + BLOCKD_TRACE("[blockd] response token mismatch dev=%u got=%u expected=%u\n", + slot->device_id, resp_pkt->token, slot->completion_token); + return; + } BLOCKD_TRACE("[blockd] backend resp token=%u dev=%u status=%d bytes=%u data_len=%u\n", resp_pkt->token, resp_pkt->device_id, resp_pkt->status, resp_pkt->bytes_transferred, resp_pkt->data_len); @@ -504,33 +601,37 @@ blockd_backend_handle_io_response( memset(&out, 0, sizeof(out)); out.token = slot->pending_req.token; out.status = resp_pkt->status; - out.bytes_transferred = resp_pkt->bytes_transferred; - out.data_len = 0; - if (out.status == IPC_BLOCK_BACKEND_STATUS_OK) { - uint64_t expect = (uint64_t)slot->pending_req.blocks * 512ULL; - if (out.bytes_transferred != expect) { - BLOCKD_TRACE("[blockd] WARN: short backend resp dev=%u token=%u bytes=%u expect=%llu\n", - slot->device_id, slot->pending_req.token, - out.bytes_transferred, - (unsigned long long)expect); - } - } - if ((slot->pending_req.flags & IPC_BLOCK_SERVICE_F_WRITE) == 0 && + out.bytes_transferred = resp_pkt->bytes_transferred; + expected = (uint64_t)slot->pending_req.blocks * slot->block_size; + write = (slot->pending_req.flags & IPC_BLOCK_SERVICE_F_WRITE) != 0; + if (out.status == IPC_BLOCK_BACKEND_STATUS_OK && + (expected == 0 || expected > IPC_BLOCK_SERVICE_MAX_DATA || + out.bytes_transferred != expected || + resp_pkt->data_len > sizeof(resp_pkt->data) || + (!write && resp_pkt->data_len != expected) || + (write && resp_pkt->data_len != 0))) { + out.status = IPC_BLOCK_BACKEND_STATUS_INVALID; + out.bytes_transferred = 0; + } + if (out.status == IPC_BLOCK_BACKEND_STATUS_OK && !write && resp_pkt->data_len > 0) { - uint32_t copy = resp_pkt->data_len; - if (copy > IPC_BLOCK_SERVICE_MAX_DATA) - copy = IPC_BLOCK_SERVICE_MAX_DATA; - memcpy(out.data, resp_pkt->data, copy); - out.data_len = copy; + memcpy(out.data, resp_pkt->data, resp_pkt->data_len); + out.data_len = resp_pkt->data_len; } slot->waiting_completion = false; + slot->completion_token = 0; #ifdef LENIX_DEBUG blockd_log_int("[blockd] backend response token=", (int)slot->pending_req.token); #endif block_service_respond(&out); blockd_backend_promote_queued(slot); - if (blockd_backend_portal != IPC_PORTAL_INVALID_HANDLE) { + if (slot->push_enabled && !blockd_backend_push_request(slot)) { + blockd_backend_fail_requests(slot, + IPC_BLOCK_BACKEND_STATUS_ERROR); + slot->in_use = false; + } else if (!slot->push_enabled && + blockd_backend_portal != IPC_PORTAL_INVALID_HANDLE) { struct ipc_block_backend_io_response ack; memset(&ack, 0, sizeof(ack)); ack.opcode = IPC_BLOCK_BACKEND_OPCODE_IO_RESPONSE; @@ -788,21 +889,16 @@ blockd_check_device_timeouts(void) uint64_t wait_ms = now - slot->waiting_since_ms; if (wait_ms > BLOCKD_BACKEND_TIMEOUT_MS) { - /* Backend timed out - clear stuck state and fail the request */ BLOCKD_TRACE("[blockd] TIMEOUT dev=%u waited=%llu ms, clearing stuck state\n", slot->device_id, (unsigned long long)wait_ms); - - struct ipc_block_service_response out; - memset(&out, 0, sizeof(out)); - out.token = slot->pending_req.token; - out.status = -110; /* ETIMEDOUT */ - out.bytes_transferred = 0; - out.data_len = 0; - slot->waiting_completion = false; - block_service_respond(&out); - - /* Promote any queued requests */ - blockd_backend_promote_queued(slot); + blockd_backend_fail_requests(slot, -110); + if (slot->push_enabled) { + slot->in_use = false; + slot->backend_portal = IPC_PORTAL_INVALID_HANDLE; + slot->push_enabled = false; + } else { + blockd_backend_promote_queued(slot); + } } else if (wait_ms > 2000) { /* Log warning if waiting more than 2s */ BLOCKD_TRACE("[blockd] WAITING dev=%u for %llu ms token=%u\n", blob - dd1a43a66b91b17e300b153ed564aeaae063c7c8 blob + e6f5efe9bd5114e9f539fc8b928fe6387a24f92a --- servers/block/ramdiskd/main.c +++ servers/block/ramdiskd/main.c @@ -330,11 +330,44 @@ ramdisk_handle_request(const struct ipc_block_backend_ ramdisk_stats.requests_completed++; } +static int +ramdisk_push_loop(portal_handle_t push_portal, + portal_handle_t blockd_portal, uint32_t device_id) +{ + struct ipc_block_backend_io_request work; + struct ipc_block_backend_io_response response; + struct pollfd pfd; + ssize_t received; + while (!ramdisk_terminate) { + pfd.fd = (int)push_portal; + pfd.events = POLLIN; + pfd.revents = 0; + if (poll(&pfd, 1, 100) <= 0) + continue; + if ((pfd.revents & POLLIN) == 0) + continue; + memset(&work, 0, sizeof(work)); + received = portal_recv(&work, sizeof(work)); + if (received != (ssize_t)sizeof(work) || + work.opcode != IPC_BLOCK_BACKEND_OPCODE_IO_REQUEST || + work.device_id != device_id) + continue; + memset(&response, 0, sizeof(response)); + response.token = work.token; + ramdisk_handle_request(&work, &response); + if (portal_send(blockd_portal, &response, sizeof(response)) != 0) + return -1; + } + return 0; +} + + int main(void) { portal_handle_t portal; + portal_handle_t push_portal; struct ipc_service_client client; struct ipc_block_backend_register reg_req = {0}; struct ipc_block_backend_register_response reg_resp = {0}; @@ -361,6 +394,7 @@ main(void) memset(&sa, 0, sizeof(sa)); sa.sa_handler = handle_sigterm; sigaction(SIGTERM, &sa, NULL); + push_portal = portal_get_bootstrap(); /* Wait for block-backend service */ DLOG("[ramdiskd] waiting for blockd backend portal (namesvc)..."); @@ -397,10 +431,11 @@ main(void) log_line("[ramdiskd] block backend portal ready"); #endif portal_handle_t namesvc = service_resolve("namesvc"); - if (namesvc == IPC_PORTAL_INVALID_HANDLE) { + if (namesvc == IPC_PORTAL_INVALID_HANDLE || + push_portal == IPC_PORTAL_INVALID_HANDLE) { DLOG("[ramdiskd] namesvc unavailable; skipping registration"); } else { - int ns_rc = namesvc_register(namesvc, "ramdiskd", portal, + int ns_rc = namesvc_register(namesvc, "ramdiskd", push_portal, IPC_PORTAL_RIGHT_SEND, staged ? IPC_NAMESVC_FLAG_STAGED : 0); if (ns_rc != 0) { @@ -444,6 +479,10 @@ main(void) #endif DLOG("[ramdiskd] sending REGISTER request..."); reg_req.opcode = IPC_BLOCK_BACKEND_OPCODE_REGISTER; + if (push_portal != IPC_PORTAL_INVALID_HANDLE) { + reg_req.flags = IPC_BLOCK_BACKEND_REGISTER_F_PUSH; + reg_req.portal = push_portal; + } reg_req.block_size = RAMDISK_BLOCK_SIZE; reg_req.block_count = ramdisk_block_count; @@ -494,12 +533,14 @@ main(void) log_line("[ramdiskd] registered backend device"); log_hex32("[ramdiskd] device_id=", (uint32_t)device_id); #endif + memset(&ramdisk_stats, 0, sizeof(ramdisk_stats)); + if ((reg_req.flags & IPC_BLOCK_BACKEND_REGISTER_F_PUSH) != 0) + return ramdisk_push_loop(push_portal, portal, device_id); memset(&fetch, 0, sizeof(fetch)); memset(&work, 0, sizeof(work)); memset(&response, 0, sizeof(response)); memset(&ack, 0, sizeof(ack)); - memset(&ramdisk_stats, 0, sizeof(ramdisk_stats)); fetch.opcode = IPC_BLOCK_BACKEND_OPCODE_IO_REQUEST; fetch.device_id = device_id; fetch.token = 1; blob - e016b6bd71e754ac8b92f98ae6faa2b0081a4edc blob + 179ba5422a010242f76425ac6d4ce63f89df6bf1 --- servers/block/virtio-blk/main.c +++ servers/block/virtio-blk/main.c @@ -174,9 +174,12 @@ static int virtio_blk_device_init(struct virtio_pci_st static int virtio_blk_transfer(struct virtio_blk_driver *drv, bool write, uint64_t lba, void *buf, uint32_t blocks); static int virtio_backend_register(struct virtio_blk_driver *drv, - struct ipc_service_client *client, uint32_t *device_id_out); + struct ipc_service_client *client, + portal_handle_t push_portal, uint32_t *device_id_out); static void virtio_backend_loop(struct virtio_blk_driver *drv, - struct ipc_service_client *client, uint32_t device_id); + struct ipc_service_client *client, + portal_handle_t push_portal, + portal_handle_t blockd_portal, uint32_t device_id); static void virtio_lock_acquire(void); static void virtio_lock_release(void); static bool virtio_log_verbose_enabled(void); @@ -756,7 +759,8 @@ out: static int virtio_backend_register(struct virtio_blk_driver *drv, - struct ipc_service_client *client, uint32_t *device_id_out) + struct ipc_service_client *client, portal_handle_t push_portal, + uint32_t *device_id_out) { struct ipc_block_backend_register req; struct ipc_block_backend_register_response resp; @@ -766,6 +770,10 @@ virtio_backend_register(struct virtio_blk_driver *drv, memset(&req, 0, sizeof(req)); memset(&resp, 0, sizeof(resp)); req.opcode = IPC_BLOCK_BACKEND_OPCODE_REGISTER; + if (push_portal != IPC_PORTAL_INVALID_HANDLE) { + req.flags = IPC_BLOCK_BACKEND_REGISTER_F_PUSH; + req.portal = push_portal; + } req.block_size = drv->block_size; req.block_count = drv->capacity; if (req.block_size == 0 || req.block_count == 0) @@ -785,12 +793,15 @@ virtio_backend_register(struct virtio_blk_driver *drv, static void virtio_backend_loop(struct virtio_blk_driver *drv, - struct ipc_service_client *client, uint32_t device_id) + struct ipc_service_client *client, portal_handle_t push_portal, + portal_handle_t blockd_portal, uint32_t device_id) { static struct ipc_block_backend_io_request fetch; static struct ipc_block_backend_io_request work; static struct ipc_block_backend_io_response response; static struct ipc_block_backend_io_response ack; + struct pollfd pfd; + ssize_t received; int busy_spins = 0; static int busy_log_count; @@ -799,38 +810,51 @@ virtio_backend_loop(struct virtio_blk_driver *drv, log_line(LOG_PREFIX "SIGTERM received, exiting backend loop"); return; } - memset(&fetch, 0, sizeof(fetch)); - fetch.opcode = IPC_BLOCK_BACKEND_OPCODE_IO_REQUEST; - fetch.device_id = device_id; - if (ipc_service_request_issue(client, &fetch, sizeof(fetch), - &work, sizeof(work)) != 0) - continue; - if (work.status == IPC_BLOCK_BACKEND_STATUS_BUSY || - work.status == IPC_BLOCK_BACKEND_STATUS_NOT_FOUND || - work.status == IPC_BLOCK_BACKEND_STATUS_IDLE) { - /* No work yet; back off briefly (longer for IDLE) */ - busy_spins++; - if (busy_spins > 10000) { - if (virtio_log_verbose_enabled() && - busy_log_count < 4) { - log_hex32(LOG_PREFIX "fetch busy status ", - (uint32_t)work.status); - busy_log_count++; + if (push_portal != IPC_PORTAL_INVALID_HANDLE) { + pfd.fd = (int)push_portal; + pfd.events = POLLIN; + pfd.revents = 0; + if (poll(&pfd, 1, 100) <= 0 || + (pfd.revents & POLLIN) == 0) + continue; + memset(&work, 0, sizeof(work)); + received = portal_recv(&work, sizeof(work)); + if (received != (ssize_t)sizeof(work) || + work.opcode != IPC_BLOCK_BACKEND_OPCODE_IO_REQUEST || + work.device_id != device_id) + continue; + } else { + memset(&fetch, 0, sizeof(fetch)); + fetch.opcode = IPC_BLOCK_BACKEND_OPCODE_IO_REQUEST; + fetch.device_id = device_id; + if (ipc_service_request_issue(client, &fetch, sizeof(fetch), + &work, sizeof(work)) != 0) + continue; + if (work.status == IPC_BLOCK_BACKEND_STATUS_BUSY || + work.status == IPC_BLOCK_BACKEND_STATUS_NOT_FOUND || + work.status == IPC_BLOCK_BACKEND_STATUS_IDLE) { + busy_spins++; + if (busy_spins > 10000) { + if (virtio_log_verbose_enabled() && + busy_log_count < 4) { + log_hex32(LOG_PREFIX "fetch busy status ", + (uint32_t)work.status); + busy_log_count++; + } + busy_spins = 0; } - busy_spins = 0; + if (work.status == IPC_BLOCK_BACKEND_STATUS_IDLE) + poll(NULL, 0, 10); + else + poll(NULL, 0, 1); + continue; } - /* Use moderate backoff for IDLE (no work pending) */ - if (work.status == IPC_BLOCK_BACKEND_STATUS_IDLE) - poll(NULL, 0, 10); - else - poll(NULL, 0, 1); - continue; + busy_spins = 0; + if (work.status != IPC_BLOCK_BACKEND_STATUS_OK) { + log_hex32(LOG_PREFIX "fetch status ", work.status); + continue; + } } - busy_spins = 0; - if (work.status != IPC_BLOCK_BACKEND_STATUS_OK) { - log_hex32(LOG_PREFIX "fetch status ", work.status); - continue; - } memset(&response, 0, sizeof(response)); response.opcode = IPC_BLOCK_BACKEND_OPCODE_IO_RESPONSE; response.token = work.token; @@ -888,9 +912,14 @@ virtio_backend_loop(struct virtio_blk_driver *drv, log_hex32(LOG_PREFIX "blocks ", blocks); log_hex32(LOG_PREFIX "token ", response.token); } - if (ipc_service_request_issue(client, &response, sizeof(response), - &ack, sizeof(ack)) != 0) + if (push_portal != IPC_PORTAL_INVALID_HANDLE) { + if (portal_send(blockd_portal, &response, + sizeof(response)) != 0) + return; + } else if (ipc_service_request_issue(client, &response, + sizeof(response), &ack, sizeof(ack)) != 0) { continue; + } } } @@ -985,6 +1014,7 @@ main(void) { struct ipc_service_client backend_client; portal_handle_t backend_portal; + portal_handle_t push_portal; static portal_handle_t cached_backend = IPC_PORTAL_INVALID_HANDLE; bool staged = srv_should_stage(); uint32_t gen = 0; @@ -998,6 +1028,7 @@ main(void) memset(&sa, 0, sizeof(sa)); sa.sa_handler = virtio_handle_sigterm; sigaction(SIGTERM, &sa, NULL); + push_portal = portal_get_bootstrap(); virtio_log_verbose = false; #ifdef LENIX_DEBUG @@ -1065,7 +1096,7 @@ main(void) log_line(LOG_PREFIX "failed to init backend client - skipping virtio-blk"); return 0; } - if (virtio_backend_register(&virtio_driver, &backend_client, + if (virtio_backend_register(&virtio_driver, &backend_client, push_portal, &backend_device_id) != 0) { log_line(LOG_PREFIX "backend registration failed - skipping virtio-blk"); return 0; @@ -1075,8 +1106,9 @@ main(void) log_hex32(LOG_PREFIX "backend device id ", backend_device_id); { portal_handle_t namesvc = service_resolve("namesvc"); - if (namesvc != IPC_PORTAL_INVALID_HANDLE) - (void)namesvc_register(namesvc, "virtio-blk", backend_portal, + if (namesvc != IPC_PORTAL_INVALID_HANDLE && + push_portal != IPC_PORTAL_INVALID_HANDLE) + (void)namesvc_register(namesvc, "virtio-blk", push_portal, IPC_PORTAL_RIGHT_SEND, staged ? IPC_NAMESVC_FLAG_STAGED : 0); } @@ -1085,6 +1117,7 @@ main(void) return 0; } log_line(LOG_PREFIX "initialization complete, entering event loop"); - virtio_backend_loop(&virtio_driver, &backend_client, backend_device_id); + virtio_backend_loop(&virtio_driver, &backend_client, push_portal, + backend_portal, backend_device_id); return 0; } blob - 42cc40383a9fb3a8cefc75783f0b6323379f73ea blob + 0a6dee99c3b6771d6ca929b05651c132912a5564 --- servers/fs/ext2/main.c +++ servers/fs/ext2/main.c @@ -26,7 +26,7 @@ #define EXT2_SYMLINK_MAX_FOLLOW 8 /* Read-ahead cache configuration */ -#define EXT2_READAHEAD_ENABLE 1 +#define EXT2_READAHEAD_ENABLE 0 #define EXT2_READAHEAD_BLOCKS 8 /* Fetch 8 blocks (8KB for 1024-byte blocks) on cache miss */ #define EXT2_ENOENT (-2) @@ -606,6 +606,15 @@ ext2_write_block(uint32_t block_no, const void *buf) /* Forward declarations */ static int ext2_is_valid_block_no(uint32_t block); +static uint64_t +ext2_group_desc_offset(void) +{ + uint32_t block; + + block = (ext2_ctx.block_size == 1024) ? 2U : 1U; + return (uint64_t)block * ext2_ctx.block_size; +} + static int __attribute__((unused)) ext2_allocate_block(uint32_t *allocated_block_no) { @@ -613,6 +622,14 @@ ext2_allocate_block(uint32_t *allocated_block_no) static uint8_t bitmap_block_buf[EXT2_MAX_BLOCK_SIZE]; uint32_t bitmap_block_no; uint32_t block_no; + uint32_t first_data_block_in_group; + uint32_t group_first_block; + uint32_t group_block_limit; + uint32_t inode_table_block; + uint32_t inode_table_blocks; + uint32_t inode_table_end; + uint32_t bitmap_bit; + uint64_t gd_offset; size_t byte_offset; size_t bit_offset; int found = 0; @@ -626,11 +643,11 @@ ext2_allocate_block(uint32_t *allocated_block_no) return -1; } - /* Hardcoded to group 0 for now - read group descriptor */ - /* Group descriptor table starts right after superblock */ - uint32_t gd_offset = 1024 + ext2_ctx.block_size; + /* Hardcoded to group 0 for now - read group descriptor. */ + gd_offset = ext2_group_desc_offset(); #ifdef LENIX_DEBUG - printf("[ext2] allocate_block: reading group descriptor from offset %u\n", gd_offset); + printf("[ext2] allocate_block: reading group descriptor from offset %llu\n", + (unsigned long long)gd_offset); #endif if (ext2_read_bytes(gd_offset, &group_desc, sizeof(struct ext2_group_desc)) != 0) { printf("[ext2] allocate_block: read group descriptor failed\n"); @@ -649,18 +666,17 @@ ext2_allocate_block(uint32_t *allocated_block_no) /* Calculate first usable data block in this group */ /* For group 0, we need to skip: superblock, group descriptors, * block bitmap, inode bitmap, and inode table */ - uint32_t first_data_block_in_group = 0; - /* Hardcoded to group 0 for now */ - uint32_t inode_table_block = group_desc.bg_inode_table; + inode_table_block = group_desc.bg_inode_table; /* Handle invalid inode table location (same logic as read_inode) */ if (inode_table_block == 0 || inode_table_block > 1024) inode_table_block = 36; /* Inode table starts at bg_inode_table, calculate where it ends */ - uint32_t inode_table_blocks = (ext2_ctx.inodes_per_group * ext2_ctx.inode_size + ext2_ctx.block_size - 1) / ext2_ctx.block_size; - uint32_t inode_table_end = inode_table_block + inode_table_blocks; + inode_table_blocks = (ext2_ctx.inodes_per_group * ext2_ctx.inode_size + + ext2_ctx.block_size - 1) / ext2_ctx.block_size; + inode_table_end = inode_table_block + inode_table_blocks; /* Data blocks start after the inode table */ first_data_block_in_group = inode_table_end; @@ -670,15 +686,26 @@ ext2_allocate_block(uint32_t *allocated_block_no) inode_table_block, inode_table_blocks, first_data_block_in_group); #endif - /* Scan bitmap for first free block (bit = 0) */ - for (uint32_t i = first_data_block_in_group; i < ext2_ctx.blocks_per_group; i++) { - byte_offset = i / 8; - bit_offset = i % 8; + group_first_block = ext2_ctx.first_data_block; + if (ext2_ctx.blocks_per_group > UINT32_MAX - group_first_block) + return -1; + group_block_limit = group_first_block + ext2_ctx.blocks_per_group; + if (group_block_limit > ext2_ctx.super.s_blocks_count) + group_block_limit = ext2_ctx.super.s_blocks_count; + if (first_data_block_in_group < group_first_block) + first_data_block_in_group = group_first_block; + /* Scan the group-0 bitmap for the first free data block. */ + for (block_no = first_data_block_in_group; + block_no < group_block_limit; block_no++) { + bitmap_bit = block_no - group_first_block; + if (bitmap_bit >= ext2_ctx.block_size * 8U) + break; + byte_offset = bitmap_bit / 8; + bit_offset = bitmap_bit % 8; + /* Check if bit is free (0 = free) */ - if ((bitmap_block_buf[byte_offset] & (1 << bit_offset)) == 0) { - /* Found free block */ - block_no = i; /* Block number within group */ + if ((bitmap_block_buf[byte_offset] & (1U << bit_offset)) == 0) { found = 1; break; } @@ -692,9 +719,10 @@ ext2_allocate_block(uint32_t *allocated_block_no) printf("[ext2] allocate_block: found free block %u in group\n", block_no); #endif /* Mark block as used in bitmap (set bit to 1) */ - byte_offset = block_no / 8; - bit_offset = block_no % 8; - bitmap_block_buf[byte_offset] |= (1 << bit_offset); + bitmap_bit = block_no - group_first_block; + byte_offset = bitmap_bit / 8; + bit_offset = bitmap_bit % 8; + bitmap_block_buf[byte_offset] |= (1U << bit_offset); /* Write bitmap back to disk */ if (ext2_write_block(bitmap_block_no, bitmap_block_buf) != 0) { @@ -744,15 +772,25 @@ ext2_free_block(uint32_t block_no) struct ext2_group_desc group_desc; static uint8_t bitmap_block_buf[EXT2_MAX_BLOCK_SIZE]; uint32_t bitmap_block_no; + uint32_t bitmap_bit; + uint32_t group_first_block; + uint64_t gd_offset; size_t byte_offset; size_t bit_offset; if (!ext2_is_valid_block_no(block_no)) return -1; - /* Hardcoded to group 0 for now - read group descriptor */ - /* Group descriptor table starts right after superblock */ - uint32_t gd_offset = 1024 + ext2_ctx.block_size; + group_first_block = ext2_ctx.first_data_block; + if (block_no < group_first_block) + return -1; + bitmap_bit = block_no - group_first_block; + if (bitmap_bit >= ext2_ctx.blocks_per_group || + bitmap_bit >= ext2_ctx.block_size * 8U) + return -1; + + /* Hardcoded to group 0 for now - read group descriptor. */ + gd_offset = ext2_group_desc_offset(); if (ext2_read_bytes(gd_offset, &group_desc, sizeof(struct ext2_group_desc)) != 0) return -1; @@ -766,9 +804,9 @@ ext2_free_block(uint32_t block_no) return -1; /* Mark block as free in bitmap (clear bit to 0) */ - byte_offset = block_no / 8; - bit_offset = block_no % 8; - bitmap_block_buf[byte_offset] &= ~(1 << bit_offset); + byte_offset = bitmap_bit / 8; + bit_offset = bitmap_bit % 8; + bitmap_block_buf[byte_offset] &= ~(1U << bit_offset); /* Write bitmap back to disk */ if (ext2_write_block(bitmap_block_no, bitmap_block_buf) != 0) @@ -1116,15 +1154,19 @@ ext2_allocate_inode(uint32_t *allocated_inode_no) struct ext2_group_desc group_desc; uint8_t bitmap_block_buf[EXT2_MAX_BLOCK_SIZE]; uint32_t bitmap_block_no; + uint32_t bitmap_bit; + uint32_t first_inode; uint32_t inode_no; + uint64_t gd_offset; size_t byte_offset, bit_offset; int found = 0; if (allocated_inode_no == NULL) return -1; - /* Hardcoded to group 0 - read group descriptor */ - if (ext2_read_bytes(ext2_ctx.first_data_block * 512 + 32, &group_desc, 32) != 0) + /* Hardcoded to group 0 - read group descriptor. */ + gd_offset = ext2_group_desc_offset(); + if (ext2_read_bytes(gd_offset, &group_desc, sizeof(group_desc)) != 0) return -1; /* Get inode bitmap block number from group descriptor */ @@ -1144,14 +1186,19 @@ ext2_allocate_inode(uint32_t *allocated_inode_no) ext2_ctx.inodes_per_group); #endif - for (uint32_t i = 11; i < ext2_ctx.inodes_per_group; i++) { - byte_offset = i / 8; - bit_offset = i % 8; + first_inode = ext2_ctx.super.s_first_ino; + if (first_inode == 0) + first_inode = 11; + for (inode_no = first_inode; + inode_no <= ext2_ctx.inodes_per_group; inode_no++) { + bitmap_bit = inode_no - 1; + if (bitmap_bit >= ext2_ctx.block_size * 8U) + break; + byte_offset = bitmap_bit / 8; + bit_offset = bitmap_bit % 8; /* Check if bit is free (0 = free) */ - if ((bitmap_block_buf[byte_offset] & (1 << bit_offset)) == 0) { - /* Found free inode */ - inode_no = i; + if ((bitmap_block_buf[byte_offset] & (1U << bit_offset)) == 0) { found = 1; #ifdef LENIX_DEBUG printf("[ext2] allocate_inode: allocated inode %u\n", inode_no); @@ -1166,9 +1213,10 @@ ext2_allocate_inode(uint32_t *allocated_inode_no) } /* Mark inode as used in bitmap (set bit to 1) */ - byte_offset = inode_no / 8; - bit_offset = inode_no % 8; - bitmap_block_buf[byte_offset] |= (1 << bit_offset); + bitmap_bit = inode_no - 1; + byte_offset = bitmap_bit / 8; + bit_offset = bitmap_bit % 8; + bitmap_block_buf[byte_offset] |= (1U << bit_offset); /* Write bitmap back to disk */ if (ext2_write_block(bitmap_block_no, bitmap_block_buf) != 0) @@ -1179,7 +1227,7 @@ ext2_allocate_inode(uint32_t *allocated_inode_no) group_desc.bg_free_inodes_count--; /* Write updated group descriptor back */ - if (ext2_write_bytes(ext2_ctx.first_data_block * 512 + 32, &group_desc, 32) != 0) + if (ext2_write_bytes(gd_offset, &group_desc, sizeof(group_desc)) != 0) return -1; /* Update superblock free inode count */ @@ -1201,13 +1249,16 @@ ext2_free_inode(uint32_t inode_no) struct ext2_group_desc group_desc; uint8_t bitmap_block_buf[EXT2_MAX_BLOCK_SIZE]; uint32_t bitmap_block_no; + uint32_t bitmap_bit; + uint64_t gd_offset; size_t byte_offset, bit_offset; - if (inode_no == 0) + if (inode_no == 0 || inode_no > ext2_ctx.inodes_per_group) return -1; - /* Hardcoded to group 0 - read group descriptor */ - if (ext2_read_bytes(ext2_ctx.first_data_block * 512 + 32, &group_desc, 32) != 0) + /* Hardcoded to group 0 - read group descriptor. */ + gd_offset = ext2_group_desc_offset(); + if (ext2_read_bytes(gd_offset, &group_desc, sizeof(group_desc)) != 0) return -1; /* Get inode bitmap block number */ @@ -1220,9 +1271,12 @@ ext2_free_inode(uint32_t inode_no) return -1; /* Mark inode as free in bitmap (clear bit to 0) */ - byte_offset = inode_no / 8; - bit_offset = inode_no % 8; - bitmap_block_buf[byte_offset] &= ~(1 << bit_offset); + bitmap_bit = inode_no - 1; + if (bitmap_bit >= ext2_ctx.block_size * 8U) + return -1; + byte_offset = bitmap_bit / 8; + bit_offset = bitmap_bit % 8; + bitmap_block_buf[byte_offset] &= ~(1U << bit_offset); /* Write bitmap back to disk */ if (ext2_write_block(bitmap_block_no, bitmap_block_buf) != 0) @@ -1233,7 +1287,7 @@ ext2_free_inode(uint32_t inode_no) group_desc.bg_free_inodes_count++; /* Write updated group descriptor back */ - if (ext2_write_bytes(ext2_ctx.first_data_block * 512 + 32, &group_desc, 32) != 0) + if (ext2_write_bytes(gd_offset, &group_desc, sizeof(group_desc)) != 0) return -1; /* Update superblock free inode count */ blob - 14148c7225887861770d8f0479b579e062e3c919 blob + 75cc22a91f0f82517da9efc8b56391fe699e673c --- servers/fs/vfs/main.c +++ servers/fs/vfs/main.c @@ -340,6 +340,7 @@ vfs_handle_reset(struct vfs_file_handle *handle) handle->path[0] = '\0'; #if VFS_READAHEAD_ENABLE handle->logical_offset = 0; + handle->backend_offset = 0; handle->cache_file_offset = -1; handle->cache_valid = 0; #endif @@ -1336,6 +1337,7 @@ vfs_init(void) handle_table[i].flags = 0; #if VFS_READAHEAD_ENABLE handle_table[i].logical_offset = 0; + handle_table[i].backend_offset = 0; handle_table[i].cache_file_offset = -1; handle_table[i].cache_valid = 0; #endif @@ -1662,36 +1664,37 @@ vfs_respond_read(const struct ipc_vfs_request *req) goto done; } - /* Seek FS fd to logical position before reading */ - #ifdef LENIX_DEBUG - printf("[vfs] seeking to pos=%lld\n", (long long)pos); - #endif - memset(&fs_request_buffer, 0, sizeof(fs_request_buffer)); - fs_request_buffer.opcode = IPC_FS_REQ_SEEK; - fs_request_buffer.body.seek.fd = handle->fs_fd; - fs_request_buffer.body.seek.offset = pos; - fs_request_buffer.body.seek.whence = SEEK_SET; - - if (ipc_service_request_issue(&fs_client_buffer, &fs_request_buffer, - sizeof(fs_request_buffer), &fs_response_buffer, - sizeof(fs_response_buffer)) != 0 || - fs_response_buffer.opcode != IPC_FS_RESP_SEEK) { + /* Synchronize the backend only after cached data changed position. */ + if (handle->backend_offset != pos) { #ifdef LENIX_DEBUG - printf("[vfs] seek failed: ipc_err or wrong opcode\n"); + printf("[vfs] seeking to pos=%lld\n", (long long)pos); #endif - if (copied == 0) { - resp.body.read.status = VFS_EIO; - resp.body.read.data_len = 0; - vfs_send_response(&resp); - return; + memset(&fs_request_buffer, 0, sizeof(fs_request_buffer)); + fs_request_buffer.opcode = IPC_FS_REQ_SEEK; + fs_request_buffer.body.seek.fd = handle->fs_fd; + fs_request_buffer.body.seek.offset = pos; + fs_request_buffer.body.seek.whence = SEEK_SET; + + if (ipc_service_request_issue(&fs_client_buffer, + &fs_request_buffer, sizeof(fs_request_buffer), + &fs_response_buffer, + sizeof(fs_response_buffer)) != 0 || + fs_response_buffer.opcode != IPC_FS_RESP_SEEK || + fs_response_buffer.body.seek.status != 0) { + #ifdef LENIX_DEBUG + printf("[vfs] seek failed\n"); + #endif + if (copied == 0) { + resp.body.read.status = VFS_EIO; + resp.body.read.data_len = 0; + vfs_send_response(&resp); + return; + } + goto done; } - goto done; + handle->backend_offset = + fs_response_buffer.body.seek.new_offset; } - #ifdef LENIX_DEBUG - printf("[vfs] seek status=%d new_offset=%lld\n", - fs_response_buffer.body.seek.status, - (long long)fs_response_buffer.body.seek.new_offset); - #endif /* Fetch read-ahead data (up to VFS_READAHEAD_SIZE bytes total) */ size_t total_fetched = 0; @@ -1710,11 +1713,14 @@ vfs_respond_read(const struct ipc_vfs_request *req) if (ipc_service_request_issue(&fs_client_buffer, &fs_request_buffer, sizeof(fs_request_buffer), &fs_response_buffer, sizeof(fs_response_buffer)) != 0 || - fs_response_buffer.opcode != IPC_FS_RESP_READ) { + fs_response_buffer.opcode != IPC_FS_RESP_READ || + fs_response_buffer.body.read.status < 0) { break; } size_t fetched = fs_response_buffer.body.read.data_len; + if (fetched > chunk) + fetched = chunk; if (fetched == 0) break; /* EOF */ @@ -1727,6 +1733,7 @@ vfs_respond_read(const struct ipc_vfs_request *req) memcpy(handle->cache + total_fetched, fs_response_buffer.body.read.data, fetched); total_fetched += fetched; + handle->backend_offset += (int64_t)fetched; if (fetched < chunk) break; /* EOF or short read */ @@ -1752,8 +1759,29 @@ vfs_respond_read(const struct ipc_vfs_request *req) vfs_send_response(&resp); return; } + if (!handle->is_device && handle->backend_offset != pos) { + memset(&fs_request_buffer, 0, sizeof(fs_request_buffer)); + memset(&fs_response_buffer, 0, sizeof(fs_response_buffer)); + fs_request_buffer.opcode = IPC_FS_REQ_SEEK; + fs_request_buffer.body.seek.fd = handle->fs_fd; + fs_request_buffer.body.seek.offset = pos; + fs_request_buffer.body.seek.whence = SEEK_SET; + if (ipc_service_request_issue(&fs_client_buffer, + &fs_request_buffer, sizeof(fs_request_buffer), + &fs_response_buffer, sizeof(fs_response_buffer)) != 0 || + fs_response_buffer.opcode != IPC_FS_RESP_SEEK || + fs_response_buffer.body.seek.status != 0) { + resp.body.read.status = VFS_EIO; + resp.body.read.data_len = 0; + vfs_send_response(&resp); + return; + } + handle->backend_offset = + fs_response_buffer.body.seek.new_offset; + } memset(&fs_request_buffer, 0, sizeof(fs_request_buffer)); + memset(&fs_response_buffer, 0, sizeof(fs_response_buffer)); fs_request_buffer.opcode = IPC_FS_REQ_READ; fs_request_buffer.body.read.fd = handle->fs_fd; fs_request_buffer.body.read.length = (uint32_t)requested_len; @@ -1761,11 +1789,15 @@ vfs_respond_read(const struct ipc_vfs_request *req) if (ipc_service_request_issue(&fs_client_buffer, &fs_request_buffer, sizeof(fs_request_buffer), &fs_response_buffer, sizeof(fs_response_buffer)) == 0 && - fs_response_buffer.opcode == IPC_FS_RESP_READ) { + fs_response_buffer.opcode == IPC_FS_RESP_READ && + fs_response_buffer.body.read.status >= 0) { copied = fs_response_buffer.body.read.data_len; + if (copied > requested_len) + copied = requested_len; if (copied > 0) { memcpy(resp.body.read.data, fs_response_buffer.body.read.data, copied); pos += copied; + handle->backend_offset += (int64_t)copied; } } } @@ -1841,6 +1873,29 @@ vfs_respond_write(const struct ipc_vfs_request *req) return; } +#if VFS_READAHEAD_ENABLE + if (!handle->is_device && + handle->backend_offset != handle->logical_offset) { + memset(&fs_request_buffer, 0, sizeof(fs_request_buffer)); + memset(&fs_response_buffer, 0, sizeof(fs_response_buffer)); + fs_request_buffer.opcode = IPC_FS_REQ_SEEK; + fs_request_buffer.body.seek.fd = handle->fs_fd; + fs_request_buffer.body.seek.offset = handle->logical_offset; + fs_request_buffer.body.seek.whence = SEEK_SET; + if (ipc_service_request_issue(&fs_client_buffer, &fs_request_buffer, + sizeof(fs_request_buffer), &fs_response_buffer, + sizeof(fs_response_buffer)) != 0 || + fs_response_buffer.opcode != IPC_FS_RESP_SEEK || + fs_response_buffer.body.seek.status != 0) { + resp.body.write.status = VFS_EIO; + resp.body.write.bytes_written = 0; + vfs_send_response(&resp); + return; + } + handle->backend_offset = fs_response_buffer.body.seek.new_offset; + } +#endif + memset(&fs_request_buffer, 0, sizeof(fs_request_buffer)); memset(&fs_response_buffer, 0, sizeof(fs_response_buffer)); fs_request_buffer.opcode = IPC_FS_REQ_WRITE; @@ -1871,6 +1926,12 @@ vfs_respond_write(const struct ipc_vfs_request *req) resp.body.write.status = fs_response_buffer.body.write.status; resp.body.write.bytes_written = fs_response_buffer.body.write.bytes_written; +#if VFS_READAHEAD_ENABLE + handle->logical_offset += resp.body.write.bytes_written; + handle->backend_offset += resp.body.write.bytes_written; + handle->cache_valid = 0; + handle->cache_file_offset = -1; +#endif vfs_send_response(&resp); } static void @@ -2038,6 +2099,7 @@ vfs_respond_lseek(const struct ipc_vfs_request *req) /* Update logical offset and invalidate cache */ if (fs_response_buffer.body.seek.status == 0) { handle->logical_offset = fs_response_buffer.body.seek.new_offset; + handle->backend_offset = fs_response_buffer.body.seek.new_offset; } handle->cache_valid = 0; handle->cache_file_offset = -1; blob - 6f79491df66c2506b7c5ff8903e0f23e2013d0fe blob + b9be4769786e986b1e739294a294fca099637c30 --- servers/fs/vfs/vfs_internal.h +++ servers/fs/vfs/vfs_internal.h @@ -53,6 +53,7 @@ struct vfs_file_handle { #if VFS_READAHEAD_ENABLE /* Read-ahead cache for sequential read optimization */ int64_t logical_offset; /* Logical file offset (what user sees) */ + int64_t backend_offset; /* Current backend file descriptor offset */ int64_t cache_file_offset; /* File offset where cache[0] starts */ size_t cache_valid; /* Valid bytes in cache */ uint8_t cache[VFS_READAHEAD_SIZE]; /* Inline cache buffer */ blob - fd1d5d3b9ec10dea64a62257e135fdb8d5da7ef7 blob + 6aa5471e3b4fbd050e150cb8fce790a24eeab0a0 --- user/apps/Makefile +++ user/apps/Makefile @@ -1,6 +1,6 @@ # SPDX-License-Identifier: ISC -APPS := hello sh ls devtest ioctltest killtest termtest socktest filetest forktest pipetest exectest dirtest cachetest bench_syscall real_read bench_fs_read bench_portal_pingpong +APPS := hello sh ls devtest ioctltest killtest termtest socktest filetest forktest pipetest exectest dirtest cachetest bench_syscall real_read bench_fs_read bench_block_read bench_portal_pingpong APP_SRCS_hello := hello/hello.c APP_OBJS_hello := $(APP_SRCS_hello:%.c=$(BUILD_DIR)/user/apps/%.o) @@ -70,6 +70,10 @@ APP_SRCS_bench_fs_read := ../bench/bench_fs_read.c APP_OBJS_bench_fs_read := $(BUILD_DIR)/user/apps/bench_fs_read.o APP_ELF_bench_fs_read := $(BUILD_DIR)/user/apps/bench_fs_read.elf APP_BIN_bench_fs_read := disk/ext2root/bin/bench_fs_read +APP_SRCS_bench_block_read := ../bench/bench_block_read.c +APP_OBJS_bench_block_read := $(BUILD_DIR)/user/apps/bench_block_read.o +APP_ELF_bench_block_read := $(BUILD_DIR)/user/apps/bench_block_read.elf +APP_BIN_bench_block_read := disk/ext2root/bin/bench_block_read APP_SRCS_bench_portal_pingpong := ../bench/bench_portal_pingpong.c APP_OBJS_bench_portal_pingpong := $(BUILD_DIR)/user/apps/bench_portal_pingpong.o APP_ELF_bench_portal_pingpong := $(BUILD_DIR)/user/apps/bench_portal_pingpong.elf @@ -95,11 +99,11 @@ CRT0_OBJ ?= $(USER_BUILD_DIR)/user/runtime/src/crt0_$( all: apps -apps: $(APP_ELF_hello) $(APP_ELF_sh) $(APP_ELF_ls) $(APP_ELF_devtest) $(APP_ELF_ioctltest) $(APP_ELF_killtest) $(APP_ELF_termtest) $(APP_ELF_socktest) $(APP_ELF_filetest) $(APP_ELF_forktest) $(APP_ELF_pipetest) $(APP_ELF_exectest) $(APP_ELF_dirtest) $(APP_ELF_cachetest) $(APP_ELF_bench_syscall) $(APP_ELF_real_read) $(APP_ELF_bench_fs_read) $(APP_ELF_bench_portal_pingpong) +apps: $(APP_ELF_hello) $(APP_ELF_sh) $(APP_ELF_ls) $(APP_ELF_devtest) $(APP_ELF_ioctltest) $(APP_ELF_killtest) $(APP_ELF_termtest) $(APP_ELF_socktest) $(APP_ELF_filetest) $(APP_ELF_forktest) $(APP_ELF_pipetest) $(APP_ELF_exectest) $(APP_ELF_dirtest) $(APP_ELF_cachetest) $(APP_ELF_bench_syscall) $(APP_ELF_real_read) $(APP_ELF_bench_fs_read) $(APP_ELF_bench_block_read) $(APP_ELF_bench_portal_pingpong) install: install-apps -install-apps: $(APP_ELF_hello) $(APP_ELF_sh) $(APP_ELF_ls) $(APP_ELF_devtest) $(APP_ELF_ioctltest) $(APP_ELF_killtest) $(APP_ELF_termtest) $(APP_ELF_socktest) $(APP_ELF_filetest) $(APP_ELF_forktest) $(APP_ELF_pipetest) $(APP_ELF_exectest) $(APP_ELF_dirtest) $(APP_ELF_cachetest) $(APP_ELF_bench_syscall) $(APP_ELF_real_read) $(APP_ELF_bench_fs_read) $(APP_ELF_bench_portal_pingpong) +install-apps: $(APP_ELF_hello) $(APP_ELF_sh) $(APP_ELF_ls) $(APP_ELF_devtest) $(APP_ELF_ioctltest) $(APP_ELF_killtest) $(APP_ELF_termtest) $(APP_ELF_socktest) $(APP_ELF_filetest) $(APP_ELF_forktest) $(APP_ELF_pipetest) $(APP_ELF_exectest) $(APP_ELF_dirtest) $(APP_ELF_cachetest) $(APP_ELF_bench_syscall) $(APP_ELF_real_read) $(APP_ELF_bench_fs_read) $(APP_ELF_bench_block_read) $(APP_ELF_bench_portal_pingpong) @mkdir -p disk/ext2root/bin cp $(APP_ELF_hello) $(APP_BIN_hello) cp $(APP_ELF_sh) $(APP_BIN_sh) @@ -118,19 +122,20 @@ install-apps: $(APP_ELF_hello) $(APP_ELF_sh) $(APP_ELF cp $(APP_ELF_bench_syscall) $(APP_BIN_bench_syscall) cp $(APP_ELF_real_read) $(APP_BIN_real_read) cp $(APP_ELF_bench_fs_read) $(APP_BIN_bench_fs_read) + cp $(APP_ELF_bench_block_read) $(APP_BIN_bench_block_read) cp $(APP_ELF_bench_portal_pingpong) $(APP_BIN_bench_portal_pingpong) $(BUILD_DIR)/user/apps/%.o: user/apps/%.c @mkdir -p $(dir $@) - $(CC) $(USER_CFLAGS) -c $< -o $@ + $(CC) $(USER_CFLAGS) -MMD -MP -MF $@.d -c $< -o $@ $(BUILD_DIR)/user/apps/bench_%.o: user/bench/bench_%.c @mkdir -p $(dir $@) - $(CC) $(USER_CFLAGS) -c $< -o $@ + $(CC) $(USER_CFLAGS) -MMD -MP -MF $@.d -c $< -o $@ $(BUILD_DIR)/user/apps/%.o: user/bench/%.c @mkdir -p $(dir $@) - $(CC) $(USER_CFLAGS) -c $< -o $@ + $(CC) $(USER_CFLAGS) -MMD -MP -MF $@.d -c $< -o $@ $(APP_ELF_hello): $(APP_OBJS_hello) @mkdir -p $(dir $@) @@ -302,6 +307,16 @@ $(APP_ELF_bench_fs_read): $(APP_OBJS_bench_fs_read) $(LD) $(USER_LDFLAGS) -o $@ $(APP_OBJS_bench_fs_read) $(RUNTIME_OBJS_GLOB); \ fi +$(APP_ELF_bench_block_read): $(APP_OBJS_bench_block_read) + @mkdir -p $(dir $@) + @if [ -f $(CRT0_OBJ) ] && [ -f $(RUNTIME_ARCHIVE) ]; then \ + $(LD) $(USER_LDFLAGS) -o $@ $(CRT0_OBJ) $(APP_OBJS_bench_block_read) $(RUNTIME_ARCHIVE); \ + elif [ -f $(RUNTIME_ARCHIVE) ]; then \ + $(LD) $(USER_LDFLAGS) -o $@ $(APP_OBJS_bench_block_read) $(RUNTIME_ARCHIVE); \ + else \ + $(LD) $(USER_LDFLAGS) -o $@ $(APP_OBJS_bench_block_read) $(RUNTIME_OBJS_GLOB); \ + fi + $(APP_ELF_bench_portal_pingpong): $(APP_OBJS_bench_portal_pingpong) @mkdir -p $(dir $@) @if [ -f $(CRT0_OBJ) ] && [ -f $(RUNTIME_ARCHIVE) ]; then \ @@ -312,5 +327,7 @@ $(APP_ELF_bench_portal_pingpong): $(APP_OBJS_bench_por $(LD) $(USER_LDFLAGS) -o $@ $(APP_OBJS_bench_portal_pingpong) $(RUNTIME_OBJS_GLOB); \ fi +-include $(foreach app,$(APPS),$(addsuffix .d,$(APP_OBJS_$(app)))) + clean: rm -rf $(BUILD_DIR)/user/apps blob - 36faa8be80a45dd12b53f2a5de610daec03b9d23 blob + c0f7581e71ada08b4349c8168d7ece7fec2b0520 --- user/bench/README.md +++ user/bench/README.md @@ -1,183 +1,100 @@ -# Lenix Microkernel Performance Benchmarks +# Lenix Benchmarks -This directory contains a suite of microbenchmarks for measuring the impact of IPC and scheduler optimizations in the Lenix microkernel. +These programs measure syscall, portal, block, and filesystem paths inside a +running Lenix guest. They are small diagnostic tools, not a general benchmark +suite. Compare results only when the build, QEMU command, debug settings, CPU +mode, storage backend, and iteration counts are the same. +## Table of Contents + +- [Benchmarks](#benchmarks) +- [Build](#build) +- [Run](#run) +- [Compare Results](#compare-results) + ## Benchmarks -### bench_syscall -**Purpose:** Measure raw syscall overhead +### `bench_syscall` -**What it does:** -- Executes `getpid()` in a tight loop (100,000 iterations) -- Measures total elapsed time and per-call cost -- Provides baseline for syscall latency +Runs five batches of one million calls for each of these syscalls: -**Expected output:** -``` -[bench_syscall] Starting syscall overhead benchmark -[bench_syscall] Iterations: 100000 -[bench_syscall] Total time: XXXXX ns -[bench_syscall] Per-syscall: XXX ns -[bench_syscall] Throughput: XXXXXXX syscalls/sec -``` +- `null_syscall()` for entry, dispatch, and return overhead +- `getpid()` for a task lookup +- `getuid()` for a credential lookup -**Interpretation:** -- Lower ns/syscall = better performance -- Baseline for comparison with optimized IPC +The program warms each path first, prints every batch, and reports the median +latency. Lower nanoseconds per call and higher calls per second are better. -### bench_fs_read -**Purpose:** Measure filesystem read performance +### `bench_block_read` -**What it does:** -- Creates a test file (10 KB) -- Performs open/read/close cycles (1,000 iterations) -- Measures total time and throughput +Reads 1 MiB from a raw block device in 4 KiB requests for five batches. It +checksums every batch and fails if the data changes between runs. The default +device is 2; pass another device number as the first argument when needed. -**Expected output:** -``` -[bench_fs_read] Starting filesystem read benchmark -[bench_fs_read] Iterations: 1000 -[bench_fs_read] Total time: XXXXX ns -[bench_fs_read] Per-iteration: XXX ns -[bench_fs_read] Total data read: XXXXXXX bytes -[bench_fs_read] Throughput: XXXXXXXX bytes/sec -``` +This test bypasses VFS and ext2, so it helps separate block-service overhead +from filesystem overhead. -**Interpretation:** -- Higher throughput = better performance -- Demonstrates batch processing efficiency -- Lower per-iteration time = better caching/locality +### `bench_fs_read` -### bench_portal_pingpong -**Purpose:** Measure portal round-trip latency +Creates `/fstest`, writes and validates 64 KiB, then reads the full file ten +times using a 4 KiB buffer. Only the repeated read phase is included in the +throughput result. The active path is: -**What it does:** -- Sends request message to server -- Server echoes back response -- Measures round-trip time (10,000 iterations) -- Demonstrates inline payload efficiency - -**Note:** Requires server spawning infrastructure (not yet fully integrated) - -**Expected output:** +```text +application -> syscall -> VFS -> ext2 -> blockd -> block device ``` -[bench_portal_pingpong] Starting portal round-trip benchmark -[bench_portal_pingpong] Iterations: 10000 -[bench_portal_pingpong] Message size: 64 bytes -[bench_portal_pingpong] Total time: XXXXX ns -[bench_portal_pingpong] Per-RTT: XXX ns -``` -**Interpretation:** -- Lower ns/RTT = better performance -- Shows impact of inline payloads -- Demonstrates batch processing efficiency +The benchmark fails on a short write, short read, unexpected EOF, or content +mismatch. -## Building and Installing +### `bench_portal_pingpong` -The benchmarks are now **fully integrated into the main Lenix build system** and are built alongside other userland applications. +Measures 10,000 portal round trips with a 64-byte message. Its peer startup is +not fully integrated, so treat it as development scaffolding until the guest +can launch both sides reliably. -### Integrated Build (Recommended) -The benchmarks are automatically built and installed when you build the system: +## Build -```bash -$ make install-apps # Builds and installs all userland apps including benchmarks -$ # Benchmarks are now in disk/ext2root/bin/bench_* -``` +The root build includes all benchmarks in the user application set: -The benchmarks are also included in the disk image when you run: -```bash -$ make package-apps # Creates rootfs with all apps (including benchmarks) +```sh +make ARCH=x86_64 install-apps +make ARCH=x86_64 iso esp ``` -## Running the Benchmarks +The installed binaries are placed under `disk/ext2root/bin/` before the ext2 +image is generated. -### Method 1: From shell within Lenix -Once the system is booted, invoke the benchmarks directly: +## Run -```bash -$ bench_syscall -$ bench_fs_read -$ bench_portal_pingpong # Framework (not yet fully integrated) +Boot Lenix and run a benchmark from the shell: + +```text +# bench_syscall +# bench_block_read +# bench_block_read 1 +# bench_fs_read ``` -### Method 2: As part of automated tests -The benchmarks can be added to boot tests or startup scripts for automatic execution during system initialization (requires configuration in system init). +Use `bench_block_read` before `bench_fs_read` when investigating storage. If the +raw block result is slow, the bottleneck is below the filesystem. If raw block +I/O is healthy but the filesystem result is slow, inspect VFS and ext2 request +counts, cache behavior, and seek traffic. -## Performance Expectations +## Compare Results -After implementing the optimizations: +Record these details with every result: -- **Inline Payloads**: 10-20% reduction in small message latency -- **Batch Processing**: 15-40% improvement in server throughput -- **CPU Locality**: 20-30% reduction in context switches during benchmark runs +- commit ID and architecture +- release or debug build +- QEMU machine, accelerator, RAM, and vCPU arguments +- Lenix SMP state +- rootfs source and block device number +- complete benchmark output -## Before/After Comparison +Run a correctness check before accepting a faster result. `bench_block_read` +must report matching checksums, and `bench_fs_read` must report a successful +64 KiB validation and the expected 655,360 timed bytes. -To measure the impact of optimizations: - -1. **Build baseline**: Without optimizations - ``` - $ make clean && make - $ # Run benchmarks and record results - ``` - -2. **Build optimized**: With all optimizations enabled - ``` - $ make clean && make - $ # Run same benchmarks - ``` - -3. **Compare results**: Calculate improvement percentages - -## Tuning Parameters - -The following constants can be adjusted in the source code: - -### bench_syscall -- `ITERATIONS` - Number of syscall invocations (default: 100,000) - -### bench_fs_read -- `ITERATIONS` - Number of open/read/close cycles (default: 1,000) -- `BUFFER_SIZE` - Read buffer size (default: 4,096 bytes) -- `BENCH_FS_PATH` (env) - Optional override for the test file path. If unset, - the benchmark tries `./benchfile`, then `/benchfile`, and finally known - read-only fallbacks like `/dev/zero` and `/bin/sh` when creation is not - permitted (streaming devices are capped to a small read size per iteration). - -### bench_portal_pingpong -- `ITERATIONS` - Number of round trips (default: 10,000) -- `MESSAGE_SIZE` - Request/response size (default: 64 bytes) - -## Design Notes - -### Syscall Benchmark -- Uses `getpid()` as a trivial syscall -- No side effects that would affect timing -- Single-threaded for consistent results - -### Filesystem Benchmark -- Uses `/tmp/benchfile` for reads -- Creates fresh file each run -- Reads entire file per iteration -- Demonstrates IPC efficiency in context of real I/O - -### Portal Benchmark -- Simulates client-server interaction -- Measures full round-trip latency -- 64-byte messages test inline payload path -- Larger messages (>128B) would test external buffers - -## Contributing - -To add new benchmarks: -1. Create `bench_NAME.c` in this directory -2. Implement `main()` returning 0 on success -3. Use standard Lenix APIs (unistd.h, time.h, stdio.h) -4. Add output prefixed with `[bench_NAME]` -5. Document in this README - -## References - -- Parent document: `PERFORMANCE_OPTIMIZATION_LOG.md` -- Performance plan: `lenix_microkernel_perf_plan.md` +The current measured reference configuration and results are recorded in the +Performance Context section of `AI_CONTEXT.md`. blob - 6a0a754703653f6c7041f7affb80153972eb6cc4 blob + 13a2a8c0f74b893bc50890a9078354edb3e3d7fc --- user/bench/bench_fs_read.c +++ user/bench/bench_fs_read.c @@ -112,6 +112,7 @@ main(void) FS_SUMMARY("[bench_fs_read] Starting filesystem read benchmark\n"); FS_SUMMARY("[bench_fs_read] Iterations: %d\n", FS_ITERATIONS); FS_SUMMARY("[bench_fs_read] Buffer size: %d bytes\n", FS_BUFSIZE); + FS_SUMMARY("[bench_fs_read] Backend: VFS -> ext2 -> blockd -> device 2\n"); /* Phase 1: create and populate the test file (untimed) */ FS_LOG("[bench_fs_read] Trying test file: %s\n", FS_TEST_PATH); @@ -253,10 +254,14 @@ main(void) #if FS_BENCH_DIAG FS_SUMMARY("[bench_fs_read] === DIAGNOSTIC BREAKDOWN ===\n"); + uint64_t phase_ns = total_open_ns + total_read_ns + total_close_ns; /* Use integer percentage (x10 for one decimal place) to avoid floating point */ - uint64_t open_pct = (bench_ns > 0) ? (1000ULL * total_open_ns / bench_ns) : 0; - uint64_t read_pct = (bench_ns > 0) ? (1000ULL * total_read_ns / bench_ns) : 0; - uint64_t close_pct = (bench_ns > 0) ? (1000ULL * total_close_ns / bench_ns) : 0; + uint64_t open_pct = (phase_ns > 0) ? + (1000ULL * total_open_ns / phase_ns) : 0; + uint64_t read_pct = (phase_ns > 0) ? + (1000ULL * total_read_ns / phase_ns) : 0; + uint64_t close_pct = (phase_ns > 0) ? + (1000ULL * total_close_ns / phase_ns) : 0; FS_SUMMARY("[bench_fs_read] Total open(): %llu ns (%llu.%llu%%)\n", (unsigned long long)total_open_ns, (unsigned long long)(open_pct / 10), @@ -269,15 +274,15 @@ main(void) (unsigned long long)total_close_ns, (unsigned long long)(close_pct / 10), (unsigned long long)(close_pct % 10)); - FS_SUMMARY("[bench_fs_read] Avg open() per iter: %llu ns (%llu ms)\n", - (unsigned long long)(total_open_ns / FS_ITERATIONS), - (unsigned long long)(total_open_ns / FS_ITERATIONS / 1000000)); + FS_SUMMARY("[bench_fs_read] Open phase: %llu ns (%llu ms)\n", + (unsigned long long)total_open_ns, + (unsigned long long)(total_open_ns / 1000000)); FS_SUMMARY("[bench_fs_read] Avg read() per iter: %llu ns (%llu ms)\n", (unsigned long long)(total_read_ns / FS_ITERATIONS), (unsigned long long)(total_read_ns / FS_ITERATIONS / 1000000)); - FS_SUMMARY("[bench_fs_read] Avg close() per iter: %llu ns (%llu ms)\n", - (unsigned long long)(total_close_ns / FS_ITERATIONS), - (unsigned long long)(total_close_ns / FS_ITERATIONS / 1000000)); + FS_SUMMARY("[bench_fs_read] Close phase: %llu ns (%llu ms)\n", + (unsigned long long)total_close_ns, + (unsigned long long)(total_close_ns / 1000000)); #endif return 0; blob - /dev/null blob + a0663fef5ac4e8c683d9d76ecc9f6e6773e9362a (mode 644) --- /dev/null +++ user/bench/bench_block_read.c @@ -0,0 +1,134 @@ +/* SPDX-License-Identifier: ISC */ +/* Lenix - Developed by lex0de (lex0de@tuta.com) */ +/* lenix/user/bench/bench_block_read.c */ + +#include +#include + +#include +#include +#include +#include + +#define BENCH_BATCHES 5 +#define BENCH_BYTES (1024U * 1024U) +#define BENCH_DEFAULT_DEVICE 2U +#define BENCH_IO_SIZE 4096U + +static uint8_t io_buffer[BENCH_IO_SIZE]; + +static uint64_t +bench_checksum(const uint8_t *buffer, size_t length, uint64_t checksum) +{ + for (size_t i = 0; i < length; i++) + checksum = (checksum ^ buffer[i]) * 1099511628211ULL; + return checksum; +} + +static uint64_t +bench_elapsed_ns(const struct timespec *start, const struct timespec *end) +{ + return (uint64_t)(end->tv_sec - start->tv_sec) * 1000000000ULL + + (uint64_t)(end->tv_nsec - start->tv_nsec); +} + +static void +bench_sort(uint64_t samples[BENCH_BATCHES]) +{ + for (size_t i = 1; i < BENCH_BATCHES; i++) { + uint64_t value = samples[i]; + size_t pos = i; + + while (pos > 0 && samples[pos - 1] > value) { + samples[pos] = samples[pos - 1]; + pos--; + } + samples[pos] = value; + } +} + +int +main(int argc, char *argv[]) +{ + struct ipc_block_info info; + uint64_t checksums[BENCH_BATCHES]; + uint64_t samples[BENCH_BATCHES]; + uint64_t blocks_per_batch; + uint32_t blocks_per_io; + uint32_t device = BENCH_DEFAULT_DEVICE; + + if (argc > 1) { + int parsed = atoi(argv[1]); + + if (parsed < 0 || (uint32_t)parsed >= IPC_BLOCK_MAX_DEVICES) { + puts("[bench_block_read] ERROR: invalid device"); + return 1; + } + device = (uint32_t)parsed; + } + if (block_get_info(device, &info) != 0) { + puts("[bench_block_read] ERROR: block_get_info failed"); + return 1; + } + if (info.block_size == 0 || info.block_size > BENCH_IO_SIZE || + BENCH_IO_SIZE % info.block_size != 0) { + puts("[bench_block_read] ERROR: unsupported block size"); + return 1; + } + blocks_per_io = BENCH_IO_SIZE / info.block_size; + blocks_per_batch = BENCH_BYTES / info.block_size; + if (blocks_per_batch == 0 || info.block_count < blocks_per_batch) { + puts("[bench_block_read] ERROR: device is too small"); + return 1; + } + + printf("[bench_block_read] Device: %u\n", device); + printf("[bench_block_read] Block size: %u bytes\n", info.block_size); + printf("[bench_block_read] Bytes per batch: %u\n", BENCH_BYTES); + for (size_t batch = 0; batch < BENCH_BATCHES; batch++) { + struct timespec start; + struct timespec end; + uint64_t checksum = 1469598103934665603ULL; + + if (clock_gettime(CLOCK_MONOTONIC, &start) != 0) { + puts("[bench_block_read] ERROR: benchmark timer failed"); + return 1; + } + for (uint64_t lba = 0; lba < blocks_per_batch; + lba += blocks_per_io) { + ssize_t received; + + received = block_read(device, lba, io_buffer, + blocks_per_io); + if (received != (ssize_t)sizeof(io_buffer)) { + printf("[bench_block_read] ERROR: read failed at LBA " + "%llu\n", (unsigned long long)lba); + return 1; + } + checksum = bench_checksum(io_buffer, sizeof(io_buffer), + checksum); + } + if (clock_gettime(CLOCK_MONOTONIC, &end) != 0) { + puts("[bench_block_read] ERROR: benchmark timer failed"); + return 1; + } + samples[batch] = bench_elapsed_ns(&start, &end); + checksums[batch] = checksum; + printf("[bench_block_read] Batch %zu: %llu ns, %llu " + "bytes/sec, checksum=%llu\n", batch + 1, + (unsigned long long)samples[batch], + (unsigned long long)(BENCH_BYTES * 1000000000ULL / + samples[batch]), (unsigned long long)checksum); + } + for (size_t batch = 1; batch < BENCH_BATCHES; batch++) { + if (checksums[batch] != checksums[0]) { + puts("[bench_block_read] ERROR: checksum mismatch"); + return 1; + } + } + bench_sort(samples); + printf("[bench_block_read] Median throughput: %llu bytes/sec\n", + (unsigned long long)(BENCH_BYTES * 1000000000ULL / + samples[BENCH_BATCHES / 2])); + return 0; +} blob - 109184356b181a1709846caa73074c995200add3 blob + 2717c5d96798d6d6e3d116890125374b23874f26 --- user/bench/bench_syscall.c +++ user/bench/bench_syscall.c @@ -1,78 +1,108 @@ /* SPDX-License-Identifier: ISC */ -/* - * Benchmark: Raw syscall overhead - * Measures the cost of a trivial syscall in a tight loop - */ +/* Benchmark raw null, identity, and credential syscall overhead. */ -#include -#include -#include +#include #include -#define ITERATIONS 1000000 -#define WARMUP_ITERATIONS 1000 +#include +#include +#include -/* Minimal syscall wrapper - prevents compiler optimization */ +#define BENCH_BATCHES 5 +#define BENCH_ITERATIONS 1000000 +#define BENCH_WARMUP 1000 + +typedef long (*bench_call_t)(void); + static long __attribute__((noinline)) -lenix_bench_null_syscall(void) +bench_getpid(void) { + return getpid(); +} + +static long __attribute__((noinline)) +bench_getuid(void) +{ + return (long)getuid(); +} + +static long __attribute__((noinline)) +bench_null(void) +{ return null_syscall(); } +static uint64_t +bench_elapsed_ns(const struct timespec *start, const struct timespec *end) +{ + return (uint64_t)(end->tv_sec - start->tv_sec) * 1000000000ULL + + (uint64_t)(end->tv_nsec - start->tv_nsec); +} + +static void +bench_sort(uint64_t samples[BENCH_BATCHES]) +{ + for (size_t i = 1; i < BENCH_BATCHES; i++) { + uint64_t value = samples[i]; + size_t pos = i; + + while (pos > 0 && samples[pos - 1] > value) { + samples[pos] = samples[pos - 1]; + pos--; + } + samples[pos] = value; + } +} + +static int +bench_run(const char *name, bench_call_t call, uint64_t *median_out) +{ + struct timespec start; + struct timespec end; + uint64_t samples[BENCH_BATCHES]; + volatile long sink = 0; + + for (int i = 0; i < BENCH_WARMUP; i++) + sink = call(); + for (size_t batch = 0; batch < BENCH_BATCHES; batch++) { + if (clock_gettime(CLOCK_MONOTONIC, &start) != 0) + return -1; + for (int i = 0; i < BENCH_ITERATIONS; i++) + sink = call(); + if (clock_gettime(CLOCK_MONOTONIC, &end) != 0) + return -1; + samples[batch] = bench_elapsed_ns(&start, &end) / + BENCH_ITERATIONS; + printf("[bench_syscall] %s batch %zu: %llu ns/call\n", name, + batch + 1, (unsigned long long)samples[batch]); + } + bench_sort(samples); + *median_out = samples[BENCH_BATCHES / 2]; + printf("[bench_syscall] %s median: %llu ns/call\n", name, + (unsigned long long)*median_out); + (void)sink; + return 0; +} + int main(void) { - struct timespec start, end; - long ns_per_call; - uint64_t elapsed_ns, throughput; - volatile long sink = 0; /* Prevents compiler optimization */ - #ifdef LENIX_DEBUG - puts("[bench_syscall] Starting syscall overhead benchmark"); - printf("[bench_syscall] Iterations: %d\n", ITERATIONS); - #endif - /* Warm up - reduce cold-start cache effects */ - for (int i = 0; i < WARMUP_ITERATIONS; i++) { - sink = lenix_bench_null_syscall(); - } + uint64_t null_median; + uint64_t getpid_median; + uint64_t getuid_median; - /* Get starting time */ - if (clock_gettime(CLOCK_MONOTONIC, &start) != 0) { - puts("[bench_syscall] ERROR: clock_gettime failed"); + printf("[bench_syscall] Batches: %d\n", BENCH_BATCHES); + printf("[bench_syscall] Iterations per batch: %d\n", + BENCH_ITERATIONS); + if (bench_run("null", bench_null, &null_median) != 0 || + bench_run("getpid", bench_getpid, &getpid_median) != 0 || + bench_run("getuid", bench_getuid, &getuid_median) != 0) { + puts("[bench_syscall] ERROR: benchmark timer failed"); return 1; } - - /* Tight loop calling minimal syscall - timer NOT inside loop */ - for (int i = 0; i < ITERATIONS; i++) { - sink = lenix_bench_null_syscall(); - } - - /* Get ending time */ - if (clock_gettime(CLOCK_MONOTONIC, &end) != 0) { - puts("[bench_syscall] ERROR: clock_gettime failed"); - return 1; - } - - /* Calculate elapsed time in nanoseconds */ - elapsed_ns = (end.tv_sec - start.tv_sec) * 1000000000ULL + - (end.tv_nsec - start.tv_nsec); - if (elapsed_ns == 0) { - puts("[bench_syscall] ERROR: elapsed time is zero"); - return 1; - } - - ns_per_call = (long)(elapsed_ns / ITERATIONS); - - /* Calculate throughput: iterations per second (64-bit to avoid overflow) */ - throughput = (ITERATIONS * 1000000000ULL) / elapsed_ns; - - printf("[bench_syscall] Total time: %llu ns\n", - (unsigned long long)elapsed_ns); - printf("[bench_syscall] Per-syscall: %ld ns\n", ns_per_call); + printf("[bench_syscall] Per-syscall: %llu ns\n", + (unsigned long long)null_median); printf("[bench_syscall] Throughput: %llu syscalls/sec\n", - (unsigned long long)throughput); - - /* Use sink to prevent optimization */ - (void)sink; - + (unsigned long long)(1000000000ULL / null_median)); return 0; } blob - d8adb60bda2a7e0ba52bfdb3c8882fcea05feb3d blob + b58643a9dd143e93bbbfeba77932f4ca855fbf3d --- user/runtime/include/sys/syscall.h +++ user/runtime/include/sys/syscall.h @@ -135,15 +135,15 @@ typedef long ssize_t; #define SYS_nanosleep 574U #define SYS_getdents 575U #define SYS_fstatat 576U -#define SYS_getcwd 590U -#define SYS_chdir 591U +#define SYS_getcwd 619U +#define SYS_chdir 620U #define SYS_fchdir 592U #define SYS_getppid 593U #define SYS_unlinkat 594U #define SYS_ftruncate 595U #define SYS_lstat 596U #define SYS_null 600U /* Null syscall for performance measurement (does nothing) */ -#define SYS_pty_get_previous 601U +#define SYS_pty_get_previous 621U #define SYS_srv_stage_start 602U #define SYS_srv_stage_commit 603U #define SYS_srv_stage_abort 604U @@ -161,6 +161,7 @@ typedef long ssize_t; #define SYS_getrlimit 616U #define SYS_setrlimit 617U #define SYS_getrusage 618U +#define SYS_MAX 621U struct syscall_result { int64_t value;