handle OpenHandle and SendMessage failures

desktop/desktop.cpp

@@ -1068,8 +1068,11 @@ void ApplicationTemporaryDestroy(InstalledApplication *application) {
 }
 
 void ApplicationInstanceCrashed(EsMessage *message) {
+	EsHandle processHandle = EsProcessOpen(message->crash.pid);
+	EsAssert(processHandle); // Since the process is paused, it cannot be removed.
+
 	EsProcessState state;
-	EsProcessGetState(message->crash.process, &state); 
+	EsProcessGetState(processHandle, &state); 
 	const char *fatalErrorString = state.crashReason.errorCode >= ES_FATAL_ERROR_COUNT ? "[unknown]" 
 		: EnumLookupNameFromValue(enumStrings_EsFatalError, state.crashReason.errorCode);
 	const char *systemCallString = state.crashReason.duringSystemCall == -1 ? "[none]" 
@@ -1114,8 +1117,8 @@ void ApplicationInstanceCrashed(EsMessage *message) {
 		}
 	}
 
-	EsProcessTerminate(message->crash.process, 1); 
+	EsProcessTerminate(processHandle, 1); 
-	EsHandleClose(message->crash.process);
+	EsHandleClose(processHandle);
 }
 
 void ApplicationProcessTerminated(uint64_t pid) {

desktop/os.header

@@ -1683,7 +1683,6 @@ struct EsMessageInstanceDestroy {
 
 struct EsMessageProcessCrash {
 	EsCrashReason reason;
-	EsHandle process;
 	uintptr_t pid;
 };
 

kernel/files.cpp

@@ -1881,9 +1881,15 @@ void FSRegisterFileSystem(KFileSystem *fileSystem) {
 	EsMessage m;
 	EsMemoryZero(&m, sizeof(EsMessage));
 	m.type = ES_MSG_REGISTER_FILE_SYSTEM;
-	m.registerFileSystem.rootDirectory = desktopProcess->handleTable.OpenHandle(fileSystem->rootDirectory, _ES_NODE_DIRECTORY_WRITE, KERNEL_OBJECT_NODE);
 	m.registerFileSystem.isBootFileSystem = fileSystem->isBootFileSystem;
-	desktopProcess->messageQueue.SendMessage(nullptr, &m);
+
+	m.registerFileSystem.rootDirectory = desktopProcess->handleTable.OpenHandle(fileSystem->rootDirectory, _ES_NODE_DIRECTORY_WRITE, KERNEL_OBJECT_NODE);
+
+	if (m.registerFileSystem.rootDirectory) {
+		if (!desktopProcess->messageQueue.SendMessage(nullptr, &m)) {
+			desktopProcess->handleTable.CloseHandle(m.registerFileSystem.rootDirectory); // This will check that the handle is still valid.
+		}
+	}
 }
 
 void FSRegisterBlockDevice(KBlockDevice *device) {

kernel/objects.cpp

@@ -71,8 +71,13 @@ struct HandleTable {
 	bool destroyed;
 	uint32_t handleCount;
 
+	// Be careful putting handles in the handle table!
+	// The process will be able to immediately close it.
+	// If this fails, the handle is closed and ES_INVALID_HANDLE is returned.
 	EsHandle OpenHandle(void *_object, uint32_t _flags, KernelObjectType _type, EsHandle at = ES_INVALID_HANDLE);
+
 	bool CloseHandle(EsHandle handle);
+	void ModifyFlags(EsHandle handle, uint32_t newFlags);
 
 	// Resolve the handle if it is valid and return the type in type.
 	// The initial value of type is used as a mask of expected object types for the handle.
@@ -438,6 +443,17 @@ bool HandleTable::CloseHandle(EsHandle handle) {
 	return true;
 }
 
+void HandleTable::ModifyFlags(EsHandle handle, uint32_t newFlags) {
+	KMutexAcquire(&lock);
+	EsDefer(KMutexRelease(&lock));
+	if ((!handle) || handle >= HANDLE_TABLE_L1_ENTRIES * HANDLE_TABLE_L2_ENTRIES) return;
+	HandleTableL2 *l2 = l1r.t[handle / HANDLE_TABLE_L2_ENTRIES];
+	if (!l2) return;
+	Handle *_handle = l2->t + (handle % HANDLE_TABLE_L2_ENTRIES);
+	if (!_handle->object) return;
+	_handle->flags = newFlags;
+}
+
 void HandleTable::ResolveHandle(KObject *object, EsHandle handle) {
 	KernelObjectType requestedType = object->type;
 	object->type = COULD_NOT_RESOLVE_HANDLE;
@@ -463,8 +479,8 @@ void HandleTable::ResolveHandle(KObject *object, EsHandle handle) {
 	EsDefer(KMutexRelease(&lock));
 
 	HandleTableL2 *l2 = l1r.t[handle / HANDLE_TABLE_L2_ENTRIES];
+	if (!l2) return;
 
-	if (l2) {
 	Handle *_handle = l2->t + (handle % HANDLE_TABLE_L2_ENTRIES);
 
 	if ((_handle->type & requestedType) && (_handle->object)) {
@@ -475,7 +491,6 @@ void HandleTable::ResolveHandle(KObject *object, EsHandle handle) {
 			object->valid = object->close = true;
 		}
 	}
-	}
 }
 
 // TODO Switch the order of flags and type, so that the default value of flags can be 0.
@@ -515,6 +530,7 @@ EsHandle HandleTable::OpenHandle(void *object, uint32_t flags, KernelObjectType
 
 		if (!l1->t[l1Index]) l1->t[l1Index] = (HandleTableL2 *) EsHeapAllocate(sizeof(HandleTableL2), true, K_FIXED);
 		HandleTableL2 *l2 = l1->t[l1Index];
+		if (!l2) goto error;
 		uintptr_t l2Index = HANDLE_TABLE_L2_ENTRIES;
 
 		for (uintptr_t i = 0; i < HANDLE_TABLE_L2_ENTRIES; i++) {
@@ -539,7 +555,7 @@ EsHandle HandleTable::OpenHandle(void *object, uint32_t flags, KernelObjectType
 	}
 
 	error:;
-	// TODO Close the handle to the object with CloseHandleToObject?
+	CloseHandleToObject(object, type, flags);
 	return ES_INVALID_HANDLE;
 }
 
@@ -578,19 +594,7 @@ ConstantBuffer *MakeConstantBuffer(K_USER_BUFFER const void *data, size_t bytes)
 
 EsHandle MakeConstantBuffer(K_USER_BUFFER const void *data, size_t bytes, Process *process) {
 	void *object = MakeConstantBuffer(data, bytes);
-
+	return object ? process->handleTable.OpenHandle(object, 0, KERNEL_OBJECT_CONSTANT_BUFFER) : ES_INVALID_HANDLE; 
-	if (!object) {
-		return ES_INVALID_HANDLE;
-	}
-
-	EsHandle h = process->handleTable.OpenHandle(object, 0, KERNEL_OBJECT_CONSTANT_BUFFER); 
-
-	if (h == ES_INVALID_HANDLE) {
-		CloseHandleToObject(object, KERNEL_OBJECT_CONSTANT_BUFFER, 0);
-		return ES_INVALID_HANDLE;
-	}
-
-	return h;
 }
 
 EsHandle MakeConstantBufferForDesktop(K_USER_BUFFER const void *data, size_t bytes) {

kernel/posix.cpp

@@ -261,25 +261,13 @@ namespace POSIX {
 				file->path = (char *) EsHeapAllocate(pathLength + 1, true, K_FIXED);
 				EsMemoryCopy(file->path, path, pathLength);
 
-				EsHandle fd = handleTable->OpenHandle(file, (flags & O_CLOEXEC) ? FD_CLOEXEC : 0, KERNEL_OBJECT_POSIX_FD);
-
-				// EsPrint("OPEN '%s' %z\n", pathLength, path, (openFlags & ES_NODE_WRITE_ACCESS) ? "Write" : ((openFlags & ES_NODE_READ_ACCESS) ? "Read" : "None"));
-
-				if (!fd) {
-					if (file->type == POSIX_FILE_NORMAL || file->type == POSIX_FILE_DIRECTORY) {
-						CloseHandleToObject(file->node, KERNEL_OBJECT_NODE, openFlags);
-					}
-
-					EsHeapFree(file->path, pathLength + 1, K_FIXED);
-					EsHeapFree(file, sizeof(POSIXFile), K_FIXED);
-					return -ENFILE;
-				}
-
 				if ((flags & O_TRUNC) && file->type == POSIX_FILE_NORMAL) {
 					FSFileResize(file->node, 0);
 				}
 
-				return fd;
+				// EsPrint("OPEN '%s' %z\n", pathLength, path, (openFlags & ES_NODE_WRITE_ACCESS) ? "Write" : ((openFlags & ES_NODE_READ_ACCESS) ? "Read" : "None"));
+
+				return handleTable->OpenHandle(file, (flags & O_CLOEXEC) ? FD_CLOEXEC : 0, KERNEL_OBJECT_POSIX_FD) ?: -ENFILE;
 			} break;
 
 			case ES_POSIX_SYSCALL_GET_POSIX_FD_PATH: {
@@ -324,30 +312,18 @@ namespace POSIX {
 					return _object1.flags;
 				} else if (syscall.arguments[1] == F_SETFD) {
 					KObject object;
-					uint64_t newFlags = syscall.arguments[2];
+					uint32_t newFlags = syscall.arguments[2];
-					handleTable->ResolveHandle(&object, syscall.arguments[0], &newFlags);
+					handleTable->ModifyFlags(syscall.arguments[0], newFlags);
 				} else if (syscall.arguments[1] == F_GETFL) {
 					return file->posixFlags;
 				} else if (syscall.arguments[1] == F_DUPFD) {
-					OpenHandleToObject(file, KERNEL_OBJECT_POSIX_FD, _object1.flags);
+					// Duplicate with FD_CLOEXEC clear.
-					int fd = handleTable->OpenHandle(_object1.object, _object1.flags, _object1.type);
+					OpenHandleToObject(file, KERNEL_OBJECT_POSIX_FD, 0);
-
+					return handleTable->OpenHandle(_object1.object, 0, _object1.type) ?: -ENFILE;
-					if (!fd) {
-						CloseHandleToObject(file, KERNEL_OBJECT_POSIX_FD, _object1.flags);
-						return -EBUSY;
-					} else return fd;
 				} else if (syscall.arguments[1] == F_DUPFD_CLOEXEC) {
-					KObject object;
+					// Duplicate with FD_CLOEXEC set.
-					uint64_t newFlags = syscall.arguments[2];
+					OpenHandleToObject(file, KERNEL_OBJECT_POSIX_FD, FD_CLOEXEC);
-					handleTable->ResolveHandle(&object, syscall.arguments[0], &newFlags);
+					return handleTable->OpenHandle(_object1.object, FD_CLOEXEC, _object1.type) ?: -ENFILE;
-
-					OpenHandleToObject(file, KERNEL_OBJECT_POSIX_FD, _object1.flags);
-					int fd = handleTable->OpenHandle(_object1.object, _object1.flags, _object1.type);
-
-					if (!fd) {
-						CloseHandleToObject(file, KERNEL_OBJECT_POSIX_FD, _object1.flags);
-						return -EBUSY;
-					} else return fd;
 				} else {
 					KernelPanic("POSIX::DoSyscall - Unimplemented fcntl %d.\n", syscall.arguments[1]);
 				}
@@ -570,10 +546,7 @@ namespace POSIX {
 
 				if (!process) return -ENOMEM;
 
-				EsHandle processHandle = currentProcess->handleTable.OpenHandle(process, 0, KERNEL_OBJECT_PROCESS);
+				return currentProcess->handleTable.OpenHandle(process, 0, KERNEL_OBJECT_PROCESS);
-				if (!processHandle) CloseHandleToObject(process, KERNEL_OBJECT_PROCESS);
-
-				return processHandle;
 			} break;
 
 			case SYS_exit_group: {
@@ -600,7 +573,6 @@ namespace POSIX {
 					KEventSet(&process->killedEvent);
 
 					processHandle = currentProcess->handleTable.OpenHandle(currentThread->posixData->forkProcess, 0, KERNEL_OBJECT_PROCESS);
-					if (!processHandle) CloseHandleToObject(currentThread->posixData->forkProcess, KERNEL_OBJECT_PROCESS);
 
 					// Close any handles the process owned.
 					process->handleTable.Destroy();
@@ -645,11 +617,8 @@ namespace POSIX {
 				// Clone the oldfd as newfd.
 
 				OpenHandleToObject(file, KERNEL_OBJECT_POSIX_FD, _object1.flags);
-
+				return handleTable->OpenHandle(_object1.object, _object1.flags, _object1.type, 
-				if (!handleTable->OpenHandle(_object1.object, _object1.flags, _object1.type, ConvertStandardInputTo3(syscall.arguments[1]))) {
+						ConvertStandardInputTo3(syscall.arguments[1])) ? 0 : -EBUSY;
-					CloseHandleToObject(file, KERNEL_OBJECT_POSIX_FD, _object1.flags);
-					return -EBUSY;
-				} else return 0;
 			} break;
 
 			case SYS_pipe2: {
@@ -665,7 +634,7 @@ namespace POSIX {
 				POSIXFile *writer = (POSIXFile *) EsHeapAllocate(sizeof(POSIXFile), true, K_FIXED);
 
 				if (!reader || !writer || !pipe) {
-					EsHeapFree(pipe, sizeof(Pipe), K_PAGED);
+					EsHeapFree(pipe, 0, K_PAGED);
 					EsHeapFree(reader, 0, K_FIXED);
 					EsHeapFree(writer, 0, K_FIXED);
 					return -ENOMEM;
@@ -683,29 +652,12 @@ namespace POSIX {
 				writer->handles = 1;
 				writer->pipe = pipe;
 
-				EsHandle fd0, fd1;
-				fd0 = handleTable->OpenHandle(reader, (syscall.arguments[1] & O_CLOEXEC) ? FD_CLOEXEC : 0, KERNEL_OBJECT_POSIX_FD);
-
-				if (!fd0) { 
-					EsHeapFree(pipe, sizeof(Pipe), K_PAGED);
-					EsHeapFree(reader, 0, K_FIXED);
-					EsHeapFree(writer, 0, K_FIXED);
-					return -EMFILE;
-				}
-
 				pipe->readers = 1;
-
-				fd1 = handleTable->OpenHandle(writer, (syscall.arguments[1] & O_CLOEXEC) ? FD_CLOEXEC : 0, KERNEL_OBJECT_POSIX_FD);
-
-				if (!fd1) { 
-					handleTable->CloseHandle(ConvertStandardInputTo3(fd0));
-					EsHeapFree(writer, 0, K_FIXED);
-					return -EMFILE;
-				}
-
 				pipe->writers = 1;
 
-				fildes[0] = fd0, fildes[1] = fd1;
+				fildes[0] = handleTable->OpenHandle(reader, (syscall.arguments[1] & O_CLOEXEC) ? FD_CLOEXEC : 0, KERNEL_OBJECT_POSIX_FD);
+				fildes[1] = handleTable->OpenHandle(writer, (syscall.arguments[1] & O_CLOEXEC) ? FD_CLOEXEC : 0, KERNEL_OBJECT_POSIX_FD);
+
 				return 0;
 			} break;
 

kernel/scheduler.cpp

@@ -894,11 +894,12 @@ void Scheduler::RemoveProcess(Process *process) {
 
 	// Free the process.
 
-	KRegisterAsyncTask([] (EsGeneric process) { 
+	KRegisterAsyncTask([] (EsGeneric _process) { 
-		MMSpace *space = ((Process *) process.p)->vmm;
+		Process *process = (Process *) _process.p;
-		MMFinalizeVAS(space);
+		MMSpace *space = process->vmm;
+		if (process->executableStartRequest) MMFinalizeVAS(space);
 		scheduler.mmSpacePool.Remove(space);
-		scheduler.processPool.Remove(process.p); 
+		scheduler.processPool.Remove(process); 
 	}, process);
 
 	if (started) {
@@ -964,11 +965,9 @@ void Scheduler::CrashProcess(Process *process, EsCrashReason *crashReason) {
 	EsMemoryCopy(&process->crashReason, crashReason, sizeof(EsCrashReason));
 
 	if (!shutdown) {
-		OpenHandleToObject(process, KERNEL_OBJECT_PROCESS);
 		_EsMessageWithObject m;
 		EsMemoryZero(&m, sizeof(m));
 		m.message.type = ES_MSG_APPLICATION_CRASH;
-		m.message.crash.process = desktopProcess->handleTable.OpenHandle(process, 0, KERNEL_OBJECT_PROCESS);
 		m.message.crash.pid = process->id;
 		EsMemoryCopy(&m.message.crash.reason, crashReason, sizeof(EsCrashReason));
 		desktopProcess->messageQueue.SendMessage(&m);

kernel/syscall.cpp

@@ -1,6 +1,5 @@
 // TODO Replace ES_ERROR_UNKNOWN with proper errors.
 // TODO Clean up the return values for system calls; with FATAL_ERRORs there should need to be less error codes returned.
-// TODO Close handles if OpenHandle fails or SendMessage fails.
 // TODO If a file system call fails with an error indicating the file system is corrupted, or a drive is failing, report the problem to the user.
 
 #ifndef IMPLEMENTATION
@@ -300,11 +299,11 @@ SYSCALL_IMPLEMENT(ES_SYSCALL_PROCESS_CREATE) {
 		SYSCALL_RETURN(ES_ERROR_UNKNOWN, false);
 	}
 
-	processInformation.handle = currentProcess->handleTable.OpenHandle(process, 0, KERNEL_OBJECT_PROCESS); 
 	processInformation.pid = process->id;
+	processInformation.mainThread.tid = process->executableMainThread->id;
 
 	processInformation.mainThread.handle = currentProcess->handleTable.OpenHandle(process->executableMainThread, 0, KERNEL_OBJECT_THREAD);
-	processInformation.mainThread.tid = process->executableMainThread->id;
+	processInformation.handle = currentProcess->handleTable.OpenHandle(process, 0, KERNEL_OBJECT_PROCESS); 
 
 	SYSCALL_WRITE(argument2, &processInformation, sizeof(EsProcessInformation));
 	SYSCALL_RETURN(ES_SUCCESS, false);
@@ -403,35 +402,6 @@ SYSCALL_IMPLEMENT(ES_SYSCALL_WINDOW_CLOSE) {
 	SYSCALL_RETURN(ES_SUCCESS, false);
 }
 
-SYSCALL_IMPLEMENT(ES_SYSCALL_WINDOW_SET_OBJECT) {
-	EmbeddedWindow *window;
-	SYSCALL_HANDLE(argument0, KERNEL_OBJECT_EMBEDDED_WINDOW, window, 1);
-
-	if (window->owner != currentProcess) {
-		// TODO Permissions.
-	}
-
-	void *old = window->apiWindow;
-	window->apiWindow = (void *) argument2;
-	__sync_synchronize();
-
-	KMutexAcquire(&windowManager.mutex);
-
-	if (window->container) {
-		EsMessage message;
-		EsMemoryZero(&message, sizeof(EsMessage));
-		message.type = ES_MSG_WINDOW_RESIZED;
-		int embedWidth = window->container->width - WINDOW_INSET * 2;
-		int embedHeight = window->container->height - WINDOW_INSET * 2 - CONTAINER_TAB_BAND_HEIGHT;
-		message.windowResized.content = ES_RECT_4(0, embedWidth, 0, embedHeight);
-		window->owner->messageQueue.SendMessage((void *) argument2, &message);
-	}
-
-	KMutexRelease(&windowManager.mutex);
-
-	SYSCALL_RETURN((uintptr_t) old, false);
-}
-
 SYSCALL_IMPLEMENT(ES_SYSCALL_WINDOW_SET_PROPERTY) {
 	uint8_t property = argument3;
 
@@ -493,11 +463,10 @@ SYSCALL_IMPLEMENT(ES_SYSCALL_WINDOW_SET_PROPERTY) {
 		Process *process;
 		SYSCALL_HANDLE(argument1, KERNEL_OBJECT_PROCESS, process, 2);
 		OpenHandleToObject(embed, KERNEL_OBJECT_EMBEDDED_WINDOW);
-		EsHandle handle = process->handleTable.OpenHandle(embed, 0, KERNEL_OBJECT_EMBEDDED_WINDOW);
 		KMutexAcquire(&windowManager.mutex);
 		embed->SetEmbedOwner(process);
 		KMutexRelease(&windowManager.mutex);
-		SYSCALL_RETURN(handle, false);
+		SYSCALL_RETURN(process->handleTable.OpenHandle(embed, 0, KERNEL_OBJECT_EMBEDDED_WINDOW), false);
 	} else if (property == ES_WINDOW_PROPERTY_RESIZE_CLEAR_COLOR) {
 		embed->resizeClearColor = argument1;
 	} else {
@@ -736,10 +705,10 @@ SYSCALL_IMPLEMENT(ES_SYSCALL_THREAD_CREATE) {
 		SYSCALL_RETURN(ES_ERROR_INSUFFICIENT_RESOURCES, false);
 	}
 
-	// Register processObject as a handle.
-	thread.handle = currentProcess->handleTable.OpenHandle(threadObject, 0, KERNEL_OBJECT_THREAD); 
 	thread.tid = threadObject->id;
 
+	thread.handle = currentProcess->handleTable.OpenHandle(threadObject, 0, KERNEL_OBJECT_THREAD); 
+
 	SYSCALL_WRITE(argument2, &thread, sizeof(EsThreadInformation));
 	SYSCALL_RETURN(ES_SUCCESS, false);
 }
@@ -1518,9 +1487,8 @@ SYSCALL_IMPLEMENT(ES_SYSCALL_SYSTEM_TAKE_SNAPSHOT) {
 		} break;
 	}
 
-	EsHandle constantBuffer = MakeConstantBuffer(buffer, bufferSize, currentProcess);
 	SYSCALL_WRITE(argument1, &bufferSize, sizeof(size_t));
-	SYSCALL_RETURN(constantBuffer, false);
+	SYSCALL_RETURN(MakeConstantBuffer(buffer, bufferSize, currentProcess), false);
 }
 
 SYSCALL_IMPLEMENT(ES_SYSCALL_PROCESS_OPEN) {
@@ -1588,7 +1556,10 @@ SYSCALL_IMPLEMENT(ES_SYSCALL_MESSAGE_DESKTOP) {
 		m.message.desktop.buffer = MakeConstantBufferForDesktop(buffer, argument1);
 		m.message.desktop.bytes = argument1;
 		m.message.desktop.windowID = window ? window->id : 0;
-		desktopProcess->messageQueue.SendMessage(&m);
+
+		if (!desktopProcess->messageQueue.SendMessage(&m)) {
+			desktopProcess->handleTable.CloseHandle(m.message.desktop.buffer); // This will check that the handle is still valid.
+		}
 	}
 
 	SYSCALL_RETURN(ES_SUCCESS, false);
@@ -1899,9 +1870,10 @@ SYSCALL_IMPLEMENT(ES_SYSCALL_CONNECTION_OPEN) {
 		SYSCALL_RETURN(ES_ERROR_INSUFFICIENT_RESOURCES, false);
 	}
 
-	connection.handle = currentProcess->handleTable.OpenHandle(netConnection, 0, KERNEL_OBJECT_CONNECTION); 
 	connection.error = ES_SUCCESS;
 
+	connection.handle = currentProcess->handleTable.OpenHandle(netConnection, 0, KERNEL_OBJECT_CONNECTION); 
+
 	SYSCALL_WRITE(argument0, &connection, sizeof(EsConnection));
 	SYSCALL_RETURN(ES_SUCCESS, false);
 }