berkus
/
essence-os
mirror of https://gitlab.com/nakst/essence
1
0
Fork 0
Code Releases Activity

building: move command prompt loop into start.script

This commit is contained in:
nakst 2022-03-19 17:18:06 +00:00
parent 3e21e0bda3
commit 3bc928b7dc
6 changed files with 136 additions and 197 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
apps/script_console.cpp
View File

@ -469,6 +469,7 @@ EXTERNAL_STUB(ExternalPersistWrite);
// TODO Functions only available in the POSIX subsystem:
EXTERNAL_STUB(ExternalConsoleGetLine);
EXTERNAL_STUB(ExternalConsoleWriteStdout);
EXTERNAL_STUB(ExternalSystemShellExecute);
EXTERNAL_STUB(ExternalSystemShellExecuteWithWorkingDirectory);
EXTERNAL_STUB(ExternalSystemShellEvaluate);

6
help/Build System.md
View File

@ -2,10 +2,10 @@
Building an operating system is tricky: there are many different components to be built that all support varying configurations, and there are lots of dependencies (including compilers and assemblers) that need to be prepared. The Essence build system has been designed to make this as much of a streamlined process as is possible. And hopefully it is a lot easier than building other operating systems from scratch (for a comparison, see the wonderful "Linux From Scratch" books).
The build system is divided into three components. `start.sh`, `util/build.c` and `util/build_core.c` ("build core").
The build system is divided into three components: `util/start.script`, `util/build.c` and `util/build_core.c` ("build core").
- `start.sh` first verifies the environment is acceptable for building Essence, and then it compiles and runs `util/build.c`.
- `util/build.c` provides a command prompt interface where the developer can issue commands to build and test project, and additionally manage configurations, emulators, ports and the source tree. It should be invoked by running `start.sh`. If you pass arguments to `start.sh`, then they will be forwarded to `util/build.c` and executed as a command; it will exit after this command completes. For example, `./start.sh b` will build the system without requiring interaction.
- `util/start.script` first verifies the environment is acceptable for building Essence. It then compiles `util/build.c`. It then enters a command prompt interface where the developer can issue commands to build and test project, and additionally manage configurations, emulators, ports and the source tree. Most commands are implemented by calling out to `util/build.c`.
- `util/build.c` implements most of the high-level operations of commands called in `util/start.sh`. However, its functionality is being slowly moved into `util/start.script`, so that it can be removed.
- Build core is responsible for actually building Essence components, and producing a finished drive image. It expects all dependencies and the toolchain to be prepared before it is executed. It is able to run within Essence itself (where dependencies and a toolchain are already provided). It has minimal interaction with the platform it is running on. It is typically invoked automatically by `util/build.c`, which passes a description of the requested build in `bin/build.ini`. This includes information of the toolchain to use, enabled options, applications and drivers to compile, fonts to bundle, installation settings (such as how large the drive image should be), and other general settings (like how many threads to compile with, and whether optimisations are enabled).
`util/build_common.h` contains common definitions shared between these components.

26
ports/port.script
View File

@ -20,7 +20,6 @@ str posixRoot;
// GCC port only:
bool buildCross #option;
bool runningMakefiles #persist;
// Musl port only:
bool forceRebuild #option;
@ -191,9 +190,6 @@ void PortGCC() {
str mpfrVersion = "4.1.0";
str mpcVersion = "1.2.1";
// Load the persistent variables.
assert PersistRead("bin/port.script.persist");
str destDir = "";
if buildCross {
@ -233,11 +229,11 @@ void PortGCC() {
// Ask the user if they want to resume an incomplete build.
bool resumeBuild = false;
if runningMakefiles {
if PathExists("bin/running_makefiles.txt") {
Log("The build system has detected a build was started, but was not completed.");
Log("Enter 'yes' to attempt to resume this build.");
resumeBuild = ConsoleGetYes();
runningMakefiles = false;
PathDelete("bin/running_makefiles.txt");
}
if !resumeBuild {
@ -385,7 +381,7 @@ void PortGCC() {
}
// Run makefiles.
runningMakefiles = true;
assert FileWriteAll("bin/running_makefiles.txt", "");
assert SystemShellExecuteWithWorkingDirectory("bin/build-binutils", "make -j %processorCount%");
assert SystemShellExecuteWithWorkingDirectory("bin/build-binutils", "make %destDir% install");
assert SystemShellExecuteWithWorkingDirectory("bin/build-gcc", "make all-gcc -j %processorCount%");
@ -397,9 +393,8 @@ void PortGCC() {
// Modify the mm_malloc.h header.
assert FileWriteAll("%crossDirectory%/lib/gcc/%toolchainPrefix%/%gccVersion%/include/mm_malloc.h", "/* removed */\n");
// Update the build configuration and compile the system.
FileWriteAll("bin/build_config.ini", "accepted_license=1\ncompiler_path=%compilerPath%\ncross_compiler_index=11\n");
SystemShellExecute("./start.sh c");
// Compile the system.
SystemShellExecute("bin/build c");
// Build libstdc++.
// TODO Waiting on GCC 11.3 to do this for the port. See https://gcc.gnu.org/bugzilla/show_bug.cgi?id=100017.
@ -418,7 +413,7 @@ void PortGCC() {
assert FileCopy("bin/mpfr-src/COPYING.LESSER", "bin/MPFR License.txt");
// Cleanup.
runningMakefiles = false;
PathDelete("bin/running_makefiles.txt");
PathDeleteRecursively("bin/build-binutils");
PathDeleteRecursively("bin/build-gcc");
PathDeleteRecursively("bin/build-mpc");
@ -643,3 +638,12 @@ void Start() {
}
}
}
void StartWithOptions(str _portName, bool _buildCross, bool _skipYesChecks, str _targetName, str _toolchainPrefix) {
portName = _portName;
buildCross = _buildCross;
skipYesChecks = _skipYesChecks;
targetName = _targetName;
toolchainPrefix = _toolchainPrefix;
Start();
}

191
util/build.c
View File

@ -5,7 +5,6 @@
#if defined(TARGET_X86_64)
#define TOOLCHAIN_PREFIX "x86_64-essence"
#define TARGET_NAME "x86_64"
#define TOOLCHAIN_HAS_RED_ZONE
#define TOOLCHAIN_HAS_CSTDLIB
#define QEMU_EXECUTABLE "qemu-system-x86_64"
#elif defined(TARGET_X86_32)
@ -49,12 +48,8 @@
#define ColorHighlight "\033[0;36m"
#define ColorNormal "\033[0m"
bool acceptedLicense;
bool automatedBuild;
bool foundValidCrossCompiler;
bool coloredOutput;
bool encounteredErrors;
bool interactiveMode;
volatile int emulatorTimeout;
volatile bool emulatorDidTimeout;
#ifdef __linux__
@ -250,7 +245,6 @@ void OutputStartOfBuildINI(FILE *f, bool forceDebugBuildOff) {
void BuildUtilities();
void DoCommand(const char *l);
void SaveConfig();
#define COMPILE_SKIP_COMPILE (1 << 1)
#define COMPILE_DO_BUILD (1 << 2)
@ -696,15 +690,6 @@ bool AddCompilerToPath() {
return true;
}
void SaveConfig() {
FILE *f = fopen("bin/build_config.ini", "wb");
fprintf(f, "accepted_license=%d\ncompiler_path=%s\n"
"cross_compiler_index=%d\n",
acceptedLicense, compilerPath,
foundValidCrossCompiler ? CROSS_COMPILER_INDEX : 0);
fclose(f);
}
const char *folders[] = {
"arch",
"apps",
@ -1024,9 +1009,7 @@ void BuildAndRun(int optimise, bool compile, int debug, int emulator, int log) {
Run(emulator, log, debug);
}
if (automatedBuild) {
exit(encounteredErrors ? 1 : 0);
}
exit(encounteredErrors ? 1 : 0);
}
void RunTests(int singleTest) {
@ -1078,7 +1061,6 @@ void RunTests(int singleTest) {
fprintf(stderr, ColorHighlight "%d/%d tests succeeded." ColorNormal "\n", successCount, successCount + failureCount);
fclose(testFailures);
unlink("root/Essence/Settings/API Tests/test.dat");
if (failureCount && automatedBuild) exit(1);
}
void DoCommand(const char *l) {
@ -1133,8 +1115,6 @@ void DoCommand(const char *l) {
BuildAndRun(OPTIMISE_FULL, true /* compile */, DEBUG_NONE /* debug */, EMULATOR_QEMU, LOG_VERBOSE);
} else if (0 == strcmp(l, "tlv")) {
BuildAndRun(OPTIMISE_ON, true /* compile */, DEBUG_LATER /* debug */, EMULATOR_QEMU, LOG_VERBOSE);
} else if (0 == strcmp(l, "exit") || 0 == strcmp(l, "x") || 0 == strcmp(l, "quit") || 0 == strcmp(l, "q")) {
exit(0);
} else if (0 == strcmp(l, "compile") || 0 == strcmp(l, "c")) {
LoadOptions();
Compile(COMPILE_FOR_EMULATOR, atoi(GetOptionString("Emulator.PrimaryDriveMB")), NULL);
@ -1316,11 +1296,6 @@ void DoCommand(const char *l) {
} else if (0 == memcmp(l, "do ", 3)) {
CallSystem(l + 3);
} else if (0 == memcmp(l, "live ", 5) || 0 == strcmp(l, "live")) {
if (interactiveMode) {
fprintf(stderr, "This command cannot be used in interactive mode. Type \"quit\" and then run \"./start.sh live <...>\".\n");
return;
}
if (argc < 4) {
fprintf(stderr, "Usage: \"./start.sh live <iso/raw> <drive size in MB> [extra options]\".\n");
return;
@ -1447,15 +1422,11 @@ void DoCommand(const char *l) {
l2 = (char *) l + 11;
}
int status = CallSystemF("bin/script ports/port.script portName=%s targetName=" TARGET_NAME " toolchainPrefix=" TOOLCHAIN_PREFIX, l2);
CallSystemF("bin/script ports/port.script portName=%s targetName=" TARGET_NAME " toolchainPrefix=" TOOLCHAIN_PREFIX, l2);
if (!alreadyNamedPort) {
free(l2);
}
if (automatedBuild) {
exit(status);
}
} else if (0 == strcmp(l, "make-crash-report")) {
system("rm crash-report.tar.gz");
system("mkdir crash-report");
@ -1501,26 +1472,8 @@ void DoCommand(const char *l) {
MAKE_TOOLCHAIN_WRAPPER("size");
MAKE_TOOLCHAIN_WRAPPER("strings");
MAKE_TOOLCHAIN_WRAPPER("strip");
foundValidCrossCompiler = true;
getcwd(compilerPath, sizeof(compilerPath) - 64);
strcat(compilerPath, "/cross/bin2");
SaveConfig();
} else if (0 == strcmp(l, "get-toolchain")) {
#if defined(__linux__) && defined(__x86_64__)
fprintf(stderr, "Downloading the compiler toolchain...\n");
CallSystem("bin/script util/get_source.script checksum=700205f81c7a5ca3279ae7b1fdb24e025d33b36a9716b81a71125bf0fec0de50 "
"directoryName=prefix url=https://github.com/nakst/build-gcc/releases/download/gcc-11.1.0/gcc-x86_64-essence.tar.xz");
DoCommand("setup-pre-built-toolchain");
AddCompilerToPath();
#else
if (automatedBuild) {
CallSystem("bin/script ports/port.script portName=gcc buildCross=true skipYesChecks=true targetName=" TARGET_NAME " toolchainPrefix=" TOOLCHAIN_PREFIX);
} else {
CallSystem("bin/script ports/port.script portName=gcc buildCross=true targetName=" TARGET_NAME " toolchainPrefix=" TOOLCHAIN_PREFIX);
}
exit(0);
#endif
} else if (0 == strcmp(l, "help") || 0 == strcmp(l, "h") || 0 == strcmp(l, "?")) {
printf(ColorHighlight "\n=== Common Commands ===\n" ColorNormal);
printf("build (b) - Build.\n");
@ -1555,140 +1508,26 @@ int main(int _argc, char **_argv) {
argc = _argc;
argv = _argv;
realpath("cross/bin", compilerPath); // TODO Don't hard code this.
sh_new_strdup(applicationDependencies);
unlink("bin/dependencies.ini");
coloredOutput = isatty(STDERR_FILENO);
systemLog = fopen("bin/Logs/system.log", "a");
if (!systemLog) systemLog = fopen("bin/Logs/system.log", "w");
if (argc == 1) {
printf(ColorHighlight "Essence Build" ColorNormal "\nPress Ctrl-C to exit.\nCross target is " ColorHighlight TARGET_NAME ColorNormal ".\n");
if (argc < 2) {
fprintf(stderr, "Error: No command specified.\n");
exit(1);
}
systemLog = fopen("bin/Logs/system.log", "w");
char buffer[4096];
buffer[0] = 0;
{
EsINIState s = { (char *) LoadFile("bin/build_config.ini", &s.bytes) };
char path[32768 + PATH_MAX + 16];
path[0] = 0;
while (EsINIParse(&s)) {
if (s.sectionBytes) continue;
EsINIZeroTerminate(&s);
INI_READ_BOOL(accepted_license, acceptedLicense);
INI_READ_BOOL(automated_build, automatedBuild);
INI_READ_STRING(compiler_path, path);
if (0 == strcmp("cross_compiler_index", s.key)) {
foundValidCrossCompiler = atoi(s.value) == CROSS_COMPILER_INDEX;
}
}
if (path[0]) {
strcpy(compilerPath, path);
strcat(path, ":");
char *originalPath = getenv("PATH");
if (strlen(originalPath) < 32768) {
strcat(path, originalPath);
setenv("PATH", path, 1);
} else {
printf("Warning: PATH too long\n");
}
}
}
if (!acceptedLicense) {
printf("\n=== Essence License ===\n\n");
CallSystem("cat LICENSE.md");
printf("\nType 'yes' to acknowledge you have read the license, or press Ctrl-C to exit.\n");
if (!GetYes()) exit(0);
acceptedLicense = true;
}
SaveConfig();
const char *runFirstCommand = NULL;
if (argc >= 2) {
char buffer[4096];
buffer[0] = 0;
for (int i = 1; i < argc; i++) {
if (strlen(argv[i]) + 1 > sizeof(buffer) - strlen(buffer)) break;
if (i > 1) strcat(buffer, " ");
strcat(buffer, argv[i]);
}
DoCommand(buffer);
return 0;
} else {
interactiveMode = true;
}
if (CallSystem("" TOOLCHAIN_PREFIX "-gcc --version > /dev/null 2>&1 ")) {
DoCommand("get-toolchain");
runFirstCommand = "b";
foundValidCrossCompiler = true;
}
SaveConfig();
if (runFirstCommand) {
DoCommand(runFirstCommand);
}
if (!foundValidCrossCompiler) {
printf("Warning: Your cross compiler appears to be out of date.\n");
printf("Please rebuild the compiler using the command " ColorHighlight "build-cross" ColorNormal " before attempting to build the OS.\n");
}
#ifdef TOOLCHAIN_HAS_RED_ZONE
{
CallSystem("" TOOLCHAIN_PREFIX "-gcc -mno-red-zone -print-libgcc-file-name > bin/valid_compiler.txt");
FILE *f = fopen("bin/valid_compiler.txt", "r");
char buffer[256];
buffer[fread(buffer, 1, 256, f)] = 0;
fclose(f);
if (!strstr(buffer, "no-red-zone")) {
printf("Error: Compiler built without no-red-zone support.\n");
exit(1);
}
unlink("bin/valid_compiler.txt");
}
#endif
printf("Enter 'help' to get a list of commands.\n");
char *prev = NULL;
while (true) {
char *l = NULL;
size_t pos = 0;
printf("\n> ");
printf(ColorHighlight);
getline(&l, &pos, stdin);
printf(ColorNormal);
if (strlen(l) == 1) {
l = prev;
if (!l) {
l = (char *) malloc(5);
strcpy(l, "help");
}
printf("(%s)\n", l);
} else {
l[strlen(l) - 1] = 0;
}
printf(ColorNormal);
fflush(stdout);
DoCommand(l);
if (prev != l) free(prev);
prev = l;
for (int i = 1; i < argc; i++) {
if (strlen(argv[i]) + 1 > sizeof(buffer) - strlen(buffer)) break;
if (i > 1) strcat(buffer, " ");
strcat(buffer, argv[i]);
}
DoCommand(buffer);
return 0;
}

10
util/script.c
View File

@ -770,6 +770,7 @@ char baseModuleSource[] = {
// Command line:
"str ConsoleGetLine() #extcall;"
"void ConsoleWriteStdout(str x) #extcall;"
"err[str] SystemGetEnvironmentVariable(str name) #extcall;"
"err[void] SystemSetEnvironmentVariable(str name, str value) #extcall;"
"bool SystemShellExecute(str x) #extcall;" // Returns true on success.
@ -789,6 +790,7 @@ int ExternalTextWeight(ExecutionContext *context, Value *returnValue);
int ExternalTextMonospaced(ExecutionContext *context, Value *returnValue);
int ExternalTextPlain(ExecutionContext *context, Value *returnValue);
int ExternalConsoleGetLine(ExecutionContext *context, Value *returnValue);
int ExternalConsoleWriteStdout(ExecutionContext *context, Value *returnValue);
int ExternalStringSlice(ExecutionContext *context, Value *returnValue);
int ExternalCharacterToByte(ExecutionContext *context, Value *returnValue);
int ExternalSystemShellExecute(ExecutionContext *context, Value *returnValue);
@ -833,6 +835,7 @@ ExternalFunction externalFunctions[] = {
{ .cName = "TextMonospaced", .callback = ExternalTextMonospaced },
{ .cName = "TextPlain", .callback = ExternalTextPlain },
{ .cName = "ConsoleGetLine", .callback = ExternalConsoleGetLine },
{ .cName = "ConsoleWriteStdout", .callback = ExternalConsoleWriteStdout },
{ .cName = "StringSlice", .callback = ExternalStringSlice },
{ .cName = "CharacterToByte", .callback = ExternalCharacterToByte },
{ .cName = "SystemShellExecute", .callback = ExternalSystemShellExecute },
@ -7059,6 +7062,13 @@ int ExternalLog(ExecutionContext *context, Value *returnValue) {
return EXTCALL_NO_RETURN;
}
int ExternalConsoleWriteStdout(ExecutionContext *context, Value *returnValue) {
(void) returnValue;
STACK_POP_STRING(entryText, entryBytes);
printf("%.*s", (int) entryBytes, (char *) entryText);
return EXTCALL_NO_RETURN;
}
int ExternalLogOpenGroup(ExecutionContext *context, Value *returnValue) {
(void) returnValue;
if (context->c->stackPointer < 1) return -1;

99
util/start.script
View File

@ -1,9 +1,31 @@
// TODO Merge util/build.c into here.
#import "ports/port.script" port;
str options #option;
str target #option;
str targetName;
str targetToolchainPrefix;
bool acceptedLicense #persist;
str compilerPath #persist;
int compilerIndex #persist;
bool runningMakefiles #persist;
void Setup(bool forAutomation) {
assert PersistRead("bin/start.script.persist");
if !forAutomation && !acceptedLicense {
Log("=== Essence License ===\n");
Log(FileReadAll("LICENSE.md"):assert());
Log("Type 'yes' to acknowledge you have read the license, or press Ctrl-C to exit.");
str input = ConsoleGetLine(); // TODO Use ConsoleGetYes.
if input != "yes" && input != "y" { return; }
acceptedLicense = true;
}
SystemShellEnableLogging(false);
assert PathCreateLeadingDirectories("bin/dependency_files");
@ -40,21 +62,84 @@ void Setup(bool forAutomation) {
target = "TARGET_X86_64";
}
bool toolchainHasRedZone = false;
if target == "TARGET_X86_64" {
targetToolchainPrefix = "x86_64-essence";
targetName = "x86_64";
toolchainHasRedZone = true;
} else if target == "TARGET_X86_32" {
targetToolchainPrefix = "i686-elf";
targetName = "x86_32";
} else {
Log("Unknown target!");
assert false;
}
assert SystemShellExecute("gcc -o bin/build -g util/build.c -pthread -DPARALLEL_BUILD -D%target% "
+ "-Wall -Wextra -Wno-missing-field-initializers");
if forAutomation {
assert FileWriteAll("bin/build_config.ini", "accepted_license=1\nautomated_build=1\n");
assert SystemShellExecute("bin/build get-toolchain");
assert FileAppend("bin/build_config.ini", "\nautomated_build=1\n");
assert FileWriteAll("bin/commit.txt", StringSlice(StringSplitByCharacter(SystemShellEvaluate("git log"), "\n", true)[0], 8, 15));
}
int currentCompilerIndex = 1;
if compilerPath == "" {
port.StartWithOptions("gcc", true, forAutomation, targetName, targetToolchainPrefix);
compilerPath = port.compilerPath;
compilerIndex = currentCompilerIndex;
assert StringContains(SystemGetEnvironmentVariable("PATH"):assert(), compilerPath);
assert SystemShellExecute("bin/build b");
} else {
str oldPath = SystemGetEnvironmentVariable("PATH"):assert();
assert !StringContains(oldPath, "::"); // If the PATH variable has a double colon in it, GCC won't build.
assert SystemSetEnvironmentVariable("PATH", compilerPath + ":" + oldPath);
}
if compilerIndex != currentCompilerIndex {
Log("Warning: Your cross compiler appears to be out of date.");
Log("Run %TextColorHighlight()%get-toolchain%TextPlain()% before continuing.");
// TODO Implement get-toolchain.
}
if toolchainHasRedZone {
assert StringContains(SystemShellEvaluate("%targetToolchainPrefix%-gcc -mno-red-zone -print-libgcc-file-name"), "no-red-zone");
}
}
void Start() {
Setup(false);
SystemShellExecute("bin/build %options%");
Log("");
PathDelete("bin/dependencies.ini");
if (options == "") {
Log("%TextColorHighlight()%Essence Build%TextPlain()%\nPress Ctrl-C to exit.");
Log("Cross target is %TextColorHighlight()%%targetName%%TextPlain()%.");
Log("Enter 'help' to get a list of commands.");
str previousCommand = "help";
bool running = true;
while running {
ConsoleWriteStdout("\n> %TextColorHighlight()%");
str command = StringTrim(ConsoleGetLine());
ConsoleWriteStdout(TextPlain());
if command == "" {
command = previousCommand;
Log("(%command%)");
}
if command == "exit" || command == "x" || command == "quit" || command == "q" {
running = false;
} else {
SystemShellExecute("bin/build %command%");
previousCommand = command;
}
}
} else {
SystemShellExecute("bin/build %options%");
}
}
void GenerateOVF() {
@ -139,7 +224,7 @@ void AutomationBuild() {
// Setup toolchain, build the system and ports.
assert SystemShellExecute("bin/build build-optimised");
assert SystemShellExecute("bin/build build-optional-ports > /dev/null");
port.StartWithOptions("all", false, true, targetName, targetToolchainPrefix);
// Copy a few sample files.
assert PathCopyRecursively("res/Sample Images", "root/Demo Content");
@ -240,7 +325,7 @@ void AutomationRunTests() {
assert FileWriteAll("bin/config.ini", "Flag.ENABLE_POSIX_SUBSYSTEM=1\n");
assert FileWriteAll("bin/extra_applications.ini", "desktop/api_tests.ini\n");
assert SystemShellExecute("bin/build build");
assert SystemShellExecute("bin/build build-port busybox");
port.StartWithOptions("busybox", false, true, targetName, targetToolchainPrefix);
assert SystemShellExecute("bin/build run-tests");
DeleteUnneededDirectoriesForDebugInfo();
PathDelete("bin/drive");