commit 79729c5d4ed16722fe7477cfc4b40df21923ddfa
from: int16h <int16h@gmail.com>
date: Fri Dec  5 13:49:03 2025 UTC

IPC: Implemented mailbox blocking (instead of polling) to eliminate idle polling and try to reduce context switches; made service-to-service request tokens globally unique so responses can't be delivered to the wrong client.

commit - b5a77f73fc3eda6264c0524368eec0c870e25e86
commit + 79729c5d4ed16722fe7477cfc4b40df21923ddfa
blob - 969385e66cba8036b54c153133e51614d8fd8fc3
blob + a1b2ba91c955c13110251a1ec90dd3c01d3f8e8b
--- changelog.md
+++ changelog.md
@@ -1,5 +1,33 @@
 # Changelog
 
+# 2025-12-05
+
+## Blocking IPC: Wait queues and sleeping receivers
+
+Implemented blocking mailbox receive to eliminate idle polling and reduce context switches from ~136K/sec to near-zero when servers are idle.
+
+### Scheduler changes [kernel/sched/task.c, kernel/include/sched/task.h]
+- Added `SCHED_STATE_BLOCKED` to task state enum for tasks waiting on IPC
+- Added `wait_next` field to `struct sched_task` for wait queue linkage
+- `sched_block_current()`: Marks current task as BLOCKED and switches to next runnable task
+- `sched_wake()`: Transitions task from BLOCKED to RUNNABLE, adds to run queue, nudges target CPU
+- `sched_task_wait_next()` / `sched_task_wait_next_set()`: Accessor functions for wait queue linkage (maintains encapsulation)
+
+### Mailbox changes [kernel/ipc/mailbox.c, kernel/include/ipc/mailbox.h]
+- Added `wait_head` and `wait_tail` fields to `struct ipc_mailbox` for FIFO wait queue of blocked receivers
+- `ipc_mailbox_recv_block()`: Blocking receive that sleeps until a message arrives; adds task to wait queue and calls `sched_block_current()` when queue is empty
+- `ipc_mailbox_send_from()`: Now wakes first blocked receiver (if any) after enqueuing a message; wake happens outside the mailbox lock to avoid lock inversion
+
+### Syscall changes [kernel/sys/syscall.c]
+- Replaced two polling loops (`for(;;) { recv; yield; }`) in `sys_portal_recv_impl()` and stdin read path with single call to `ipc_mailbox_recv_block()`
+- Removes ~136K context switches/second when idle (servers sleep instead of busy-polling)
+
+### Architecture
+- Wait queue is FIFO: first blocked receiver is first to be woken
+- Wake happens outside mailbox lock: prevents potential deadlock with scheduler locks
+- Deadlock detection (`deadlock_recv_start/end`) still wraps blocking receive
+- No changes to userland API: blocking is transparent to portal_recv callers
+
 # 2025-12-04
 
 ## Block service hardening: timeouts, validation, and adaptive back-off
blob - 25bcb26a7511a2c9ca6a8a094461427f5e1844fc
blob + a3d8253f070118b5925b01f52437386d02d17f50
--- kernel/include/ipc/mailbox.h
+++ kernel/include/ipc/mailbox.h
@@ -71,6 +71,9 @@ struct ipc_mailbox {
 	uint64_t remote_messages_sent;
 	uint64_t remote_messages_failed;
 	uint64_t remote_bytes_sent;
+	/* wait queue for blocked receivers */
+	struct sched_task *wait_head;
+	struct sched_task *wait_tail;
 };
 
 void ipc_mailbox_init(struct ipc_mailbox *, struct sched_task *);
@@ -78,6 +81,7 @@ int ipc_mailbox_send(struct ipc_mailbox *, const void 
 int ipc_mailbox_send_from(struct ipc_mailbox *, const void *buf, size_t len,
     struct sched_task *sender, uint32_t portal_id);
 struct ipc_message *ipc_mailbox_recv(struct ipc_mailbox *);
+struct ipc_message *ipc_mailbox_recv_block(struct ipc_mailbox *);
 void ipc_mailbox_free(struct ipc_message *);
 void ipc_mailbox_drain(struct ipc_mailbox *);
 void ipc_mailbox_set_remote(struct ipc_mailbox *, uint32_t node_id, bool remote);
blob - 110f5b299de9fb14e191a84e2644ef652084048f
blob + 56fb98ba790d8bf0303025dbc80b293776abff90
--- kernel/include/sched/task.h
+++ kernel/include/sched/task.h
@@ -181,3 +181,13 @@ void	sched_task_dma_release_all(struct sched_task *);
 bool	sched_task_dma_slot_available(const struct sched_task *);
 const char *sched_task_cwd(const struct sched_task *, size_t *);
 int	sched_task_set_cwd(struct sched_task *, const char *, size_t);
+
+/*
+ * Blocking IPC support: block current task until woken
+ */
+void	sched_block_current(void);
+void	sched_wake(struct sched_task *);
+
+/* Wait queue linkage for IPC blocking */
+struct sched_task *sched_task_wait_next(const struct sched_task *);
+void	sched_task_wait_next_set(struct sched_task *, struct sched_task *);
blob - 93dd9e7027c73eb4c00f3f7be3f0ab7049c6ff1e
blob + 9c61c131f4c3cb54b40b88f3d69cd9b3d3734e90
--- kernel/ipc/mailbox.c
+++ kernel/ipc/mailbox.c
@@ -137,6 +137,8 @@ ipc_mailbox_init(struct ipc_mailbox *mbox, struct sche
 	mbox->remote_messages_sent = 0;
 	mbox->remote_messages_failed = 0;
 	mbox->remote_bytes_sent = 0;
+	mbox->wait_head = NULL;
+	mbox->wait_tail = NULL;
 }
 
 int
@@ -249,7 +251,22 @@ ipc_mailbox_send_from(struct ipc_mailbox *mbox, const 
 		mbox->depth++;
 	if (mbox->depth > mbox->peak_depth)
 		mbox->peak_depth = mbox->depth;
+
+	/* Check for blocked receivers and wake one if present */
+	struct sched_task *waiter = NULL;
+	if (mbox->wait_head != NULL) {
+		waiter = mbox->wait_head;
+		mbox->wait_head = sched_task_wait_next(waiter);
+		if (mbox->wait_head == NULL)
+			mbox->wait_tail = NULL;
+		sched_task_wait_next_set(waiter, NULL);
+	}
 	mailbox_unlock(mbox);
+
+	/* Wake the blocked receiver outside the lock */
+	if (waiter != NULL)
+		sched_wake(waiter);
+
 	perf_mailbox_depth_sample(mbox->depth);
 	perf_counter_inc(PERF_COUNTER_IPC_MAILBOX_ENQUEUE, 1);
 	return 0;
@@ -279,6 +296,60 @@ ipc_mailbox_recv(struct ipc_mailbox *mbox)
 	return msg;
 }
 
+/*
+ * ipc_mailbox_recv_block - blocking receive that sleeps until a message arrives.
+ * If no message is available, the calling task is put to sleep until a sender
+ * wakes it up by calling ipc_mailbox_send_from().
+ */
+struct ipc_message *
+ipc_mailbox_recv_block(struct ipc_mailbox *mbox)
+{
+	struct ipc_message *msg;
+	struct sched_task *task;
+
+	if (mbox == NULL)
+		return NULL;
+
+	for (;;) {
+		mailbox_lock(mbox);
+		msg = mbox->head;
+		if (msg != NULL) {
+			/* Message available - dequeue it */
+			mbox->head = msg->next;
+			if (mbox->head == NULL)
+				mbox->tail = NULL;
+			if (mbox->depth > 0)
+				mbox->depth--;
+			mailbox_unlock(mbox);
+			msg->next = NULL;
+			perf_counter_inc(PERF_COUNTER_IPC_MAILBOX_DEQUEUE, 1);
+			return msg;
+		}
+
+		/* No message - add ourselves to wait queue and block */
+		task = sched_current_task();
+		if (task == NULL) {
+			mailbox_unlock(mbox);
+			return NULL;
+		}
+
+		sched_task_wait_next_set(task, NULL);
+		if (mbox->wait_tail == NULL) {
+			mbox->wait_head = task;
+			mbox->wait_tail = task;
+		} else {
+			sched_task_wait_next_set(mbox->wait_tail, task);
+			mbox->wait_tail = task;
+		}
+		mailbox_unlock(mbox);
+
+		/* Block until woken by sender */
+		sched_block_current();
+
+		/* After waking, loop back to try receiving again */
+	}
+}
+
 void
 ipc_mailbox_free(struct ipc_message *msg)
 {
blob - ab71e8cb79bdd5e5edf61e1a1ecde7537bc17431
blob + 820a09a5d0ce833ca1851c8e6d434605cfe83829
--- kernel/ipc/service_client.c
+++ kernel/ipc/service_client.c
@@ -26,7 +26,28 @@ struct pending_service_request {
 };
 
 static spinlock_t pending_requests_lock = SPINLOCK_INIT;
+static spinlock_t service_token_lock = SPINLOCK_INIT;
+static uint32_t service_token_global = 1;
 
+/*
+ * Allocate a globally unique token for service-to-service requests.
+ * Tokens are shared across all clients targeting any portal to avoid
+ * cross-client collisions in the pending registry.
+ */
+static uint32_t
+service_next_token(void)
+{
+	uint32_t token;
+
+	spinlock_lock(&service_token_lock);
+	token = service_token_global++;
+	if (service_token_global == 0)
+		service_token_global = 1;
+	spinlock_unlock(&service_token_lock);
+
+	return token;
+}
+
 static struct pending_service_request pending_requests[PENDING_SERVICE_REQUESTS_MAX];
 
 static void
@@ -163,9 +184,8 @@ ipc_service_request_issue_timeout(struct ipc_service_c
 
 	slot->in_use = true;
 	slot->ready = false;
-	slot->token = client->next_token++;
-	if (client->next_token == 0)
-		client->next_token = 1;
+	slot->token = service_next_token();
+	client->next_token = slot->token;
 
 	/* Register this request in the global pending registry */
 	struct pending_service_request *preq = pending_service_request_alloc();
blob - 29e1953aad0e1d7866a4c0f8bf9b70280a272e97
blob + 09321e725d34d96ef760f81d4e6fe0c25db7b359
--- kernel/sched/task.c
+++ kernel/sched/task.c
@@ -58,12 +58,14 @@ enum sched_state {
 	SCHED_STATE_RUNNABLE,
 	SCHED_STATE_RUNNING,
 	SCHED_STATE_STOPPED,
+	SCHED_STATE_BLOCKED,	/* Blocked waiting for IPC */
 };
 
 struct sched_task {
 	struct sched_task *next;
 	struct sched_task *prev_global;
 	struct sched_task *next_global;
+	struct sched_task *wait_next;	/* Next task in wait queue */
 	struct sched_context ctx;
 	void (*entry)(void *);
 	void *arg;
@@ -704,6 +706,96 @@ sched_yield(void)
 }
 
 /*
+ * sched_block_current - block the current task until woken by sched_wake().
+ * The task is removed from the run queue and will not be scheduled until
+ * explicitly woken. Must be called with a valid reason for blocking set up.
+ */
+void
+sched_block_current(void)
+{
+	uint64_t flags;
+	struct sched_cpu_state *state;
+	struct sched_task *task;
+
+	if (!scheduler_started)
+		return;
+	flags = arch_irq_save();
+	state = sched_state_current();
+	if (state == NULL || !state->idle_ready) {
+		arch_irq_restore(flags);
+		return;
+	}
+	task = state->current;
+	if (task == NULL || task->is_idle) {
+		arch_irq_restore(flags);
+		return;
+	}
+	/* Mark task as blocked - it will NOT be requeued */
+	task->state = SCHED_STATE_BLOCKED;
+	/* Switch to next runnable task without requeuing current */
+	sched_switch(false);
+	arch_irq_restore(flags);
+}
+
+/*
+ * sched_wake - wake a blocked task and make it runnable again.
+ * The task will be added to its preferred CPU's run queue.
+ */
+void
+sched_wake(struct sched_task *task)
+{
+	struct sched_cpu_state *state;
+	uint32_t cpu;
+
+	if (task == NULL)
+		return;
+	if (task->state != SCHED_STATE_BLOCKED)
+		return;
+
+	/* Mark as runnable */
+	task->state = SCHED_STATE_RUNNABLE;
+	task->wait_next = NULL;
+
+	/* Add to appropriate CPU's run queue */
+	cpu = task->last_cpu;
+	state = sched_state_for_cpu(cpu);
+	if (state == NULL)
+		state = sched_state_current();
+	if (state == NULL)
+		return;
+
+	spinlock_lock(&state->runq_lock);
+	task->next = NULL;
+	if (state->run_tail == NULL) {
+		state->run_head = task;
+		state->run_tail = task;
+	} else {
+		state->run_tail->next = task;
+		state->run_tail = task;
+	}
+	state->run_length++;
+	spinlock_unlock(&state->runq_lock);
+
+	/* Nudge the CPU if it might be idle */
+	sched_nudge_cpu(cpu);
+}
+
+struct sched_task *
+sched_task_wait_next(const struct sched_task *task)
+{
+	if (task == NULL)
+		return NULL;
+	return task->wait_next;
+}
+
+void
+sched_task_wait_next_set(struct sched_task *task, struct sched_task *next)
+{
+	if (task != NULL)
+		task->wait_next = next;
+}
+
+/*
  * sched_task_trampoline - first instruction executed by every fresh task.
  * Enables interrupts, runs the entrypoint, and panics on unexpected return.
  */
blob - 9aa8273115bae10fb61c4e9c83f4e8b19803944b
blob + 943c3555f1b7ef047fed76c00460d3ed76f483ba
--- kernel/sys/syscall.c
+++ kernel/sys/syscall.c
@@ -1391,12 +1391,7 @@ sys_read_impl(uint64_t fd, uint64_t buf_addr, uint64_t
 		if (mbox == NULL)
 			return syscall_make_result(-SYSCALL_EBADF, 0);
 		deadlock_recv_start(task, 5000);
-		for (;;) {
-			msg = ipc_mailbox_recv(mbox);
-			if (msg != NULL)
-				break;
-			sched_yield();
-		}
+		msg = ipc_mailbox_recv_block(mbox);
 		deadlock_recv_end(task);
 		if (msg == NULL)
 			return syscall_make_result(-SYSCALL_EFAULT, 0);
@@ -4435,12 +4430,7 @@ sys_portal_recv_impl(uint64_t buf_addr, uint64_t len, 
 	task = sched_current_task();
 	sched_task_set_last_ipc_sender(task, NULL, IPC_PORTAL_INVALID_HANDLE);
 	deadlock_recv_start(task, 5000);  /* 5 second timeout */
-	for (;;) {
-		msg = ipc_mailbox_recv(mbox);
-		if (msg != NULL)
-			break;
-		sched_yield();
-	}
+	msg = ipc_mailbox_recv_block(mbox);
 	deadlock_recv_end(task);
 
 	/* Validate message structure before dereferencing fields.