begin collating documentation

desktop/os.header

@@ -2134,17 +2134,20 @@ function void *EsMemoryMapObject(EsHandle object, uintptr_t offset, size_t size,
 // Standard functions.
 
 function void EsAssertionFailure(EsCString cFile, int line); 
-function EsCalculationValue EsCalculateFromUserExpression(EsCString cExpression); // For user input only; do not rely on consistent behaviour across versions; use with message mutex.
-function double EsDoubleParse(STRING string, char **endptr); 
+
 function uint8_t EsRandomU8(); 
 function uint64_t EsRandomU64();
-function int64_t EsIntegerParse(STRING text); // Parses in hexadecimal if the first two characters are '0x'.
+
+function EsCalculationValue EsCalculateFromUserExpression(EsCString cExpression); // For user input only; do not rely on consistent behaviour across versions; use with message mutex.
+
 function_not_in_kernel void EsPanic(EsCString format, ...); 
 function_not_in_kernel void EsPrint(EsCString format, ...);
 function void EsPrintDirect(STRING string); 
 function void EsPrintHelloWorld(); 
+
 function void EsRandomAddEntropy(uint64_t x); 
 function void EsRandomSeed(uint64_t x); 
+
 function EsRectangle EsRectangleAdd(EsRectangle a, EsRectangle b);
 function EsRectangle EsRectangleAddBorder(EsRectangle rectangle, EsRectangle border);
 function EsRectangle EsRectangleBounding(EsRectangle a, EsRectangle b);
@@ -2159,6 +2162,7 @@ function EsRectangle EsRectangleTranslate(EsRectangle a, EsRectangle b);
 function bool EsRectangleEquals(EsRectangle a, EsRectangle b);
 function bool EsRectangleContains(EsRectangle a, int32_t x, int32_t y);
 function bool EsRectangleContainsAll(EsRectangle parent, EsRectangle child); // Returns true iff the child rectangle is entirely contained within parent.
+
 function void EsSort(void *_base, size_t nmemb, size_t size, EsComparisonCallback compar, EsGeneric argument); 
 function void EsSortWithSwapCallback(void *_base, size_t nmemb, size_t size, EsComparisonCallback compar, EsGeneric argument, EsSwapCallback swap); 
 
@@ -2293,6 +2297,9 @@ function bool EsStringEndsWith(STRING string, STRING prefix, bool caseInsensitiv
 function char *EsStringZeroTerminate(STRING string); // Free with EsHeapFree.
 function bool EsUTF8IsValid(const char *input, ptrdiff_t bytes); // Does not check for surrogate characters or overlong sequences of non-ASCII characters.
 
+function double EsDoubleParse(STRING string, char **endptr); 
+function int64_t EsIntegerParse(STRING text); // Parses in hexadecimal if the first two characters are '0x'.
+
 // CRT functions.
 
 function int EsCRTabs(int n); 

help/API Documentation.md

@@ -89,7 +89,7 @@ Threads are used to execute code in parallel. Threads can be created and termina
 
 `EsThreadGetID` gets the ID of a thread from its handle. 
 
-`EsThreadTerminate` instructs a thread to terminate. If the thread is executing privileged code at the time of the request, it will complete the prviledged code before terminating. If a thread is waiting on a synchronisation object, such as a mutex or event, it will stop waiting and terminate regardless. **Note**: if a thread owns a mutex or spinlock when it is terminated, it will **not** release the object.
+`EsThreadTerminate` instructs a thread to terminate. If the thread is executing privileged code at the time of the request, it will complete the prviledged code before terminating. If a thread is waiting on a synchronisation object, such as a mutex or event, it will stop waiting and terminate regardless. **Note**: if a thread owns a mutex or spinlock when it is terminated, it will **not** release the object. You should ensure that a thread releases all synchronisation objects still needed by the process before it terminates.
 
 A thread can always use the handle `ES_CURRENT_THREAD` to access itself. This handle should not be closed.
 
@@ -135,6 +135,8 @@ The `EsMutex` structure contains a mutex. It should be initialised to zero. When
 
 To acquire a mutex, call `EsMutexAcquire` with a pointer to the mutex. To release a mutex, call `EsMutexRelease`.  A thread must not attempt to acquire a mutex it already owns, and it must not attempt to release a mutex it does not own.
 
+When a thread is waiting for a mutex to be released so that it may acquire it, the thread will not consume CPU time. This allows other threads to run instead, or if there are none, then it gives the CPU an opportunity to reduce its power consumption. As a side effect, this means there is some delay after a thread waits for a mutex before it wakes up and continues executing. If this behaviour is undesirable, consider using `EsSpinlock` instead.
+
 ## Example
 
 In this example, the function `IncrementCount` can safely be called on different threads at the same time.
@@ -202,6 +204,29 @@ To explain why, suppose thread A executes the first pattern and thread B execute
 - Thread B attempts to acquire mutex X. Mutex X is owned by thread A, so thread B starts waiting for thread A to release it.
 - Both threads will continue to wait indefinitely for the other to perform an operation.
 
+# Spinlocks
+
+## Definitions
+
+```c
+struct EsSpinlock {
+	volatile uint8_t state;
+}
+
+void EsSpinlockAcquire(EsSpinlock *spinlock); 
+void EsSpinlockRelease(EsSpinlock *spinlock); 
+```
+
+## Description
+
+A spinlock is a synchronisation primitive. Threads can *acquire* and *release* it. Only one thread can have acquired the spinlock at a time. Before another thread can acquire it, the original thread must release it. When a thread tries to acquire an already-acquired spinlock, it will wait until the spinlock is released, and then proceed to acquire it.
+
+The `EsSpinlock` structure contains a spinlock. It should be initialised to zero. There is no need to perform any special destruction of a spinlock.
+
+To acquire a spinlock, call `EsSpinlockAcquire` with a pointer to the spinlock. To release a spinlock, call `EsSpinlockRelease`.  A thread must not attempt to acquire a spinlock it already owns, and it must not attempt to release a spinlock it does not own.
+
+When a thread is waiting for a spinlock to be released so that it may acquire it, the thread will "spin"; that is, it will continue executing, constantly checking whether the spinlock has been released so that it may acquire it. This consumes CPU power, and prevents other threads from running instead. However, this means that as soon as the spinlock becomes available a thread may instantly acquire it and continue with its work. Spinlocks are best used in situations where the code they synchronise is small. If this behaviour is undesirable, consider using `EsMutex` instead.
+
 # INI files
 
 ## Definitions

help/blog posts.txt

@@ -0,0 +1,9 @@
+A look at various methods of automatic layouting for graphical user interfaces. A bit out of date for the current API.
+https://essence.handmade.network/blog/p/3435-automatic_ui_layouts_part_1
+https://essence.handmade.network/blog/p/3436-automatic_ui_layouts_part_2
+
+Details about how the vector renderer generates outlines for arbitrary bezier paths.
+https://essence.handmade.network/blog/p/7388-generating_polygon_outlines
+
+A descent into the madness of how cylinder-head-sector addressing "works" on floppy disks and old hard disks.
+https://essence.handmade.network/blog/p/7924-cylinder-head-sector_addressing

help/notes from discord.txt

@@ -0,0 +1,290 @@
+Here is a collection of various posts about how Essence works, taken from Discord.
+
+Please note that a lot of this information contains internal implementation details which must not be relied on by application developers.
+
+= How to debug Essence running in an emulator =
+
+nakst
+ — 
+07/06/2021
+Basic steps to debug:
+1. Install GDB if you haven't already.
+2. Optional, but recommended: install gf from github.com/nakst/gf .
+3. Open GDB or gf (at the root of the essence source tree)
+4. Load the debug symbol file for your application. It will be located in "bin/<your application name>". e.g. to load the symbols for File Manager, run file "bin/File Manager"
+5. Within the Essence build system, run d. This will compile in debug mode and launch Qemu in a paused state.
+6. In GDB run target remote :1234, or in gf press the shortcut key F4. This will connect to Qemu.
+7. Set any breakpoints now (for some reason any breakpoints set in applications while Qemu is running will be ignored)
+8. In GDB run c, or in gf press the shortcut key F5. This begins execution.
+9. Debug your application as normal using GDB/gf.
+10. If you're using gf, and GDB crashes or its connection with Qemu gets wonky, you can try restarting GDB with the shortcut Ctrl+R. You'll have to press F4 to reconnect.
+
+= About networking card driver interface =
+
+nakst
+ — 
+01/27/2021
+The kernel tells drivers to send a packet with the transmit callback function pointer in NetInterface devices, which has the signature:
+
+bool (*transmit)(NetInterface *self, void *dataVirtual, uintptr_t dataPhysical, size_t dataBytes); 
+
+dataPhysical is 2KB aligned.
+once driver is done with buffer, it calls NetTransmitBufferReturn(dataVirtual); 
+There is no userland API yet, since I only just started working on networking.
+
+= About the Hello World example application =
+
+nakst
+ — 
+03/05/2021
+
+    What is a “window” in this api, just a generic UI container?
+
+A window is the top-level container. Most instances only have one window, the tab they're associated with.
+
+    does EsMessageReceive() yield to the operating system or is it non-blocking?
+
+It's blocking. However there are ongoing animations, then it will either not block, or block with a timeout specified by the animating elements.
+
+    what's the rationale behind waiting for the operating system to tell you to create yourself?
+
+There is not necessarily a one-to-one correspondence between instances and processes. For example, the system may start a process of your application to search-index a file, or to cleanup any cached files on the file system. One process can also have multiple instances, so you may receive multiple ES_MSG_INSTANCE_CREATE messages (although this is configurable on a per-application basis).
+
+    why does EsInstanceCreate need the message to make the Instance, is it just to prevent applications from creating themselves early or is there important information encoded in the EsMessage struct?
+
+Important information is in the message. An application can request an instance of an application to be started with EsApplicationStart; this then has to go via Desktop to create the instance, and then message can be sent to the started application.
+
+    can an app instance have more than 1 window or 0 windows?
+
+Each instance has one tab. But it can create additional helper windows, e.g. the inspector window I demoed in 
+project-showcase.
+
+    what's the lifetime of the returned EsMessage
+
+The message received from EsMessageReceive is valid until the next call to EsMessageReceive. 
+
+= About why resizing windows without visual artefacts is difficult =
+
+nakst
+ — 
+06/17/2021
+Synchronisation issues. The rendering and layout of a window involves multiple processes. You also have to deal with other processes updating their windows and wanting the window manager to display them. You have to deal with slow applications (i.e. there must be timeouts on everything). There is interaction between the kernel and applications, and the kernel is not allowed to block on application processes or you could DOS the system. etc.
+
+= About the build system architecture =
+
+nakst
+ — 
+07/06/2021
+No, it's actually even weirder. ./start.sh builds bin/build; bin/build builds the compiler toolchain and bin/build_core; and bin/build_core actually builds Essence.
+
+= About various aspects of the kernel =
+
+nakst
+ — 
+07/06/2021
+
+    Does the kernel offer anything like kernel modules?
+
+Currently there is support for dynamically loading kernel modules, but they cannot be unloaded until shutdown.
+
+    How does multitasking threads/processes work?
+
+Could you be more specific? The kernel uses fairly standard preemptive multitasking.
+
+    is there any way to give more exclusive access to the OS
+
+Not currently, but I would be happy to add such an API in the future! However, since there is so much less stuff running in the background in Essence, it might not be necessary :P
+
+    Passing data across processes, what are the options?
+
+There's shared memory (EsMemoryShare/EsMemoryOpen), classic Unix-like pipes, event objects for synchronisation, and message passing. 
+
+nakst
+ — 
+07/06/2021
+Each process is made of:
+- a memory space 
+- a handle table
+- a permission bitset
+- and one or more threads 
+Each thread is just a stack + register file. 
+
+= About file permissions =
+
+nakst
+ — 
+09/06/2021
+All the architectural changes are simplifications :)
+For example, there are no file system permissions. Instead, to get access to a file/folder your process has to be given a handle to it. In fact, the kernel has no concepts of mountpoints. Instead when a drive is connected, it gives a single handle of the root directory to the root process, which can then distribute it to its child processes. And the drive is unmounted when all handles to files on it are closed
+
+nakst
+ — 
+09/06/2021
+When started, an application is passed a handle to its local storage folder. It uses this handle to open handles to any files and folders within.
+
+nakst
+ — 
+09/06/2021
+It's exposed through the API by the "mountpoint" |Settings:/
+
+nakst
+ — 
+09/06/2021
+Though what's actually happening is the API selects the correct handle to use to use as the base directory for the open file system call
+
+nakst
+ — 
+09/06/2021
+The system manages installed applications, and as such, what paths each application is allowed to access. When the application is started, it is provided with all the necessary handles it needs, and any temporary ones are given on demand.
+This is all done behind the scenes though; an application doesn't need to worry about the details.
+
+= On setting the wallpaper from the build system =
+
+nakst
+ — 
+09/23/2021
+In your source tree, copy the res/Sample Images into root
+Then append something like General.wallpaper=0:/Sample Images/Waves.jpg into bin/config.ini
+
+= On using libc =
+
+nakst
+ — 
+09/29/2021
+You have 2 options:
+1. Before including essence.h, define ES_CRT_WITHOUT_PREFIX and a bunch of functions from the C standard library will be available (abs acosf asinf atan2 atan2f atanf atoi bsearch calloc cbrt cbrtf ceil ceilf cos cosf exp exp2f fabs fabsf floor floorf fmod fmodf free getenv isalpha isdigit isnanf isspace isupper isxdigit log2 log2f malloc memchr memcmp memcpy memmove memset pow powf qsort rand realloc sin sinf snprintf sprintf sqrt sqrtf strcat strchr strcmp strcpy strdup strerror strlen strncmp strncpy strnlen strstr strtol strtoul tolower vsnprintf)
+2. Alternatively, if you need a full C standard library, you can link with musl by putting with_cstdlib=1 in the [build] section of your application's .ini file. But beware, this isn't fully supported; a few things like threads won't work correctly. 
+
+= On finding executable symbols =
+
+nakst
+ — 
+09/30/2021
+No, debug info should be available. In your bin/ folder there is an executable named after the application which contains symbols, and another with a .nosymbols suffix, which does not contain symbols.
+
+(Author's note: the .nosymbols files are now located in "bin/Stripped Executables")
+
+= On debugging userland application crashes =
+
+Brett
+ — 
+09/30/2021
+Is there a breakpoint I can set to catch the error before essence kills the process or I just have to step until then?
+Brett
+ — 
+09/30/2021
+Found the crash!
+nakst
+ — 
+09/30/2021
+It's a bit messy. First you need to load the kernel symbols with add-symbol-file bin/Kernel. Then you can break in Scheduler::CrashProcess, currently scheduler.cpp:967 and then run set process->crashed=1, and you can step out of the crash handler back into your code.
+
+= On not being able to view global variables =
+
+nakst
+ — 
+09/30/2021
+This can happen if the page containing the variable has not been mapped yet. This probably means it's part of the memory mapped executable. Once your code has accessed the variable once it should be accessible from gdb
+
+= On application and kernel permissions =
+
+nakst
+ — 
+10/09/2021
+There are 2 types of permissions, kernel permissions and application permissions.
+
+Each process has a 64-bit bitmask of kernel permissions. When it creates a process, it chooses a subset of those permission bits to give to the new process. It cannot give permission bits it does not have to the new process. Kernel permissions have the prefix ES_PERMISSION_. They are passed in EsProcessCreationArguments::permissions and stored in Process::permissions.
+
+Each installed application has a 64-bit bitmask of application permissions. These are currently decided when the application is installed. When an application is launched, the application permissions are used to figure out (a) what kernel permissions the process should get (b) what handles the application should be given access to and (c) which services provided by Desktop the application can access. 
+
+For example, APPLICATION_PERMISSION_ALL_FILES means that the application will be given handles to all the mounted file systems, and it is allowed to query information about which documents are open (a service provided by Desktop), and will also be sent notifications when new file systems are connected.
+
+Important note: permission to access devices and file systems are managed through handles, rather than permission bits or user IDs. So, if you want to open a file handle you need to have a handle to an ancestor folder, which you can only get from a more privileged process. It's done this way to reduce the amount of global permission state (like POSIX's read write execute bits on every file); instead a process gets to precisely decide what a process it creates will have access to. 
+
+= What is the object in _EsMessageWithObject =
+
+nakst
+ — 
+10/09/2021
+It's used to store the relevant window for messages sent by the window manager (e.g. mouse or keyboard input), so that the receiving process can forward the message to the window. (The naming is a bit unclear because the internals of messaging has changed a bit over the years)
+
+= On accessing the layout inspector =
+
+nakst
+ — 
+10/09/2021
+Shift+right click on a tab
+
+= On the window transfer press system call =
+
+nakst
+ — 
+10/09/2021
+When you press a mouse button down the window the cursor's over becomes "pressed". Until you release the mouse button, all the mouse movement messages are sent to the pressed window. This is needed for things like e.g. dragging a scrollbar -- even if your cursor leaves the window you still want the scrollbar to keep dragging.
+However, sometimes the thing you are dragging changes window, at which point the press needs to be "transferred" to a different window. This is currently only used for dragging tabs in and out of container windows.
+Important note: this is all just an implementation detail, I might completely change out this works in the future
+
+= On constant buffers =
+
+nakst
+ — 
+10/09/2021
+Yes, it's a form of shared memory. Once created, its contents cannot be modified. But other processes can read it if they have a handle.
+
+= On adding new applications =
+
+nakst
+ — 
+10/15/2021
+The easiest way to cross-compile is using the build system. You make a manifest file, for example:
+
+[general]
+name=<the name of your application>
+[build]
+source=<absolute path to the first source file>
+source=<absolute path to the second source file>
+; etc...
+
+
+(This is for C++.)
+
+Then make the file bin/extra_applications.ini and put in it the absolute path to the manifest file.
+
+nakst
+ — 
+12/06/2021
+Yes, you all the builtin applications are cross compiled this way. You can see sample applications in apps/samples of the source tree. To add them to the list of applications to be built, you can add the path to its ini file to bin/extra_applications.ini (attached mine for example)
+
+apps/samples/list.ini
+apps/samples/hello.ini
+apps/samples/game_loop.ini
+apps/samples/converter.ini
+
+nakst
+ — 
+12/06/2021
+yes, the build system will detect it automatically from the extra_applications.ini file
+
+= On custom compile commands for other programming languages = 
+
+nakst
+ — 
+12/13/2021
+In your application's ini file you need to set a custom compile command, e.g.
+
+[build]
+custom_compile_command=./build_my_application.sh
+
+It must output an ELF binary into the bin folder with the same name as the application (the name field in [general]). 
+
+= On the API table =
+
+nakst
+ — 
+Yesterday at 8:15 AM
+You do not need to link to anything. It is a header only library.
+
+nakst
+ — 
+Yesterday at 8:44 AM
+They are not linked to any libraries. They call into the system api via the api table, which is an array of function pointers at a fixed memory address. The api header wraps these function pointers with macros, as needed.

shared/common.cpp

@@ -1139,10 +1139,12 @@ int64_t EsIntegerParse(const char *text, ptrdiff_t bytes) {
 	if (bytes == -1) bytes = EsCStringLength(text);
 
 	int base = 10;
+	bool seenFirstRecognisedCharacter = false;
 
 	if (bytes > 2 && text[0] == '0' && text[1] == 'x') {
 		text += 2, bytes -= 2;
 		base = 16;
+		seenFirstRecognisedCharacter = true;
 	}
 
 	const char *end = text + bytes;
@@ -1153,19 +1155,21 @@ int64_t EsIntegerParse(const char *text, ptrdiff_t bytes) {
 	while (text < end) {
 		char c = *text;
 
-		if (c == '-') {
+		if (c == '-' && !seenFirstRecognisedCharacter) {
 			negative = true;
-		}
-
-		if (c >= '0' && c <= '9') {
+			seenFirstRecognisedCharacter = true;
+		} else if (c >= '0' && c <= '9') {
 			result *= base;
 			result += c - '0';
+			seenFirstRecognisedCharacter = true;
 		} else if (c >= 'A' && c <= 'F' && base == 16) {
 			result *= base;
 			result += c - 'A' + 10;
+			seenFirstRecognisedCharacter = true;
 		} else if (c >= 'a' && c <= 'f' && base == 16) {
 			result *= base;
 			result += c - 'a' + 10;
+			seenFirstRecognisedCharacter = true;
 		}
 
 		text++;