rtdl: Return null when non-elf file is dlopened

options/elf/include/elf.h

@@ -572,12 +572,13 @@ typedef struct {
 #define EM_PPC64		21
 #define EM_S390			22
 #define EM_ARM			40
-#define EM_SH				42
-#define EM_SPARCV9	43
+#define EM_SH			42
+#define EM_SPARCV9		43
 #define EM_IA_64		50
 #define EM_X86_64		62
-#define EM_BLACKFIN	106
-#define EM_AARCH64	183
+#define EM_BLACKFIN		106
+#define EM_AARCH64		183
+#define EM_RISCV		243
 
 /* Linux notes this value as being interim; however applications are using this (Qt6), so we define it here. */
 #define EM_ALPHA		0x9026

options/rtdl/aarch64/elf.hpp

@@ -2,7 +2,8 @@
 
 #include <elf.h>
 
-#define ELF_CLASS 64
+#define ELF_CLASS ELFCLASS64
+#define ELF_MACHINE EM_AARCH64
 
 using elf_ehdr = Elf64_Ehdr;
 using elf_phdr = Elf64_Phdr;

options/rtdl/generic/linker.cpp

@@ -64,23 +64,23 @@ size_t tlsMaxAlignment = 16;
 // part of the global scope is considered for symbol resolution.
 uint64_t rtsCounter = 2;
 
-void seekOrDie(int fd, int64_t offset) {
+bool trySeek(int fd, int64_t offset) {
 	off_t noff;
-	if(mlibc::sys_seek(fd, offset, SEEK_SET, &noff))
-		__ensure(!"sys_seek() failed");
+	return mlibc::sys_seek(fd, offset, SEEK_SET, &noff) == 0;
 }
 
-void readExactlyOrDie(int fd, void *data, size_t length) {
+bool tryReadExactly(int fd, void *data, size_t length) {
 	size_t offset = 0;
 	while(offset < length) {
 		ssize_t chunk;
 		if(mlibc::sys_read(fd, reinterpret_cast<char *>(data) + offset,
 				length - offset, &chunk))
-			__ensure(!"sys_read() failed");
+			return false;
 		__ensure(chunk > 0);
 		offset += chunk;
 	}
 	__ensure(offset == length);
+	return true;
 }
 
 void closeOrDie(int fd) {
@@ -157,7 +157,7 @@ SharedObject *ObjectRepository::injectStaticObject(frg::string_view name,
 	return object;
 }
 
-SharedObject *ObjectRepository::requestObjectWithName(frg::string_view name,
+frg::expected<LinkerError, SharedObject *> ObjectRepository::requestObjectWithName(frg::string_view name,
 		SharedObject *origin, Scope *localScope, bool createScope, uint64_t rts) {
 	if (auto obj = findLoadedObject(name))
 		return obj;
@@ -258,8 +258,12 @@ SharedObject *ObjectRepository::requestObjectWithName(frg::string_view name,
 	auto object = frg::construct<SharedObject>(getAllocator(),
 		name.data(), std::move(chosenPath), false, localScope, rts);
 
-	_fetchFromFile(object, fd);
+	auto result = _fetchFromFile(object, fd);
 	closeOrDie(fd);
+	if(!result) {
+		frg::destruct(getAllocator(), object);
+		return result.error();
+	}
 
 	_parseDynamic(object);
 
@@ -269,7 +273,7 @@ SharedObject *ObjectRepository::requestObjectWithName(frg::string_view name,
 	return object;
 }
 
-SharedObject *ObjectRepository::requestObjectAtPath(frg::string_view path,
+frg::expected<LinkerError, SharedObject *> ObjectRepository::requestObjectAtPath(frg::string_view path,
 		Scope *localScope, bool createScope, uint64_t rts) {
 	// TODO: Support SONAME correctly.
 	auto lastSlash = path.find_last('/') + 1;
@@ -295,10 +299,16 @@ SharedObject *ObjectRepository::requestObjectAtPath(frg::string_view path,
 	frg::string<MemoryAllocator> no_prefix(getAllocator(), path);
 
 	int fd;
-	if(mlibc::sys_open((no_prefix + '\0').data(), O_RDONLY, 0, &fd))
-		return nullptr; // TODO: Free the SharedObject.
-	_fetchFromFile(object, fd);
+	if(mlibc::sys_open((no_prefix + '\0').data(), O_RDONLY, 0, &fd)) {
+		frg::destruct(getAllocator(), object);
+		return LinkerError::notFound;
+	}
+	auto result = _fetchFromFile(object, fd);
 	closeOrDie(fd);
+	if(!result) {
+		frg::destruct(getAllocator(), object);
+		return result.error();
+	}
 
 	_parseDynamic(object);
 
@@ -401,23 +411,38 @@ void ObjectRepository::_fetchFromPhdrs(SharedObject *object, void *phdr_pointer,
 }
 
 
-void ObjectRepository::_fetchFromFile(SharedObject *object, int fd) {
+frg::expected<LinkerError, void> ObjectRepository::_fetchFromFile(SharedObject *object, int fd) {
 	__ensure(!object->isMainObject);
 
 	// read the elf file header
 	elf_ehdr ehdr;
-	readExactlyOrDie(fd, &ehdr, sizeof(elf_ehdr));
+	if(!tryReadExactly(fd, &ehdr, sizeof(elf_ehdr)))
+		return LinkerError::fileTooShort;
+
+	if(ehdr.e_ident[0] != 0x7F
+			|| ehdr.e_ident[1] != 'E'
+			|| ehdr.e_ident[2] != 'L'
+			|| ehdr.e_ident[3] != 'F')
+		return LinkerError::notElf;
 
-	__ensure(ehdr.e_ident[0] == 0x7F
-			&& ehdr.e_ident[1] == 'E'
-			&& ehdr.e_ident[2] == 'L'
-			&& ehdr.e_ident[3] == 'F');
-	__ensure(ehdr.e_type == ET_EXEC || ehdr.e_type == ET_DYN);
+	if((ehdr.e_type != ET_EXEC && ehdr.e_type != ET_DYN)
+			|| ehdr.e_machine != ELF_MACHINE
+			|| ehdr.e_ident[EI_CLASS] != ELF_CLASS)
+		return LinkerError::wrongElfType;
 
 	// read the elf program headers
 	auto phdr_buffer = (char *)getAllocator().allocate(ehdr.e_phnum * ehdr.e_phentsize);
-	seekOrDie(fd, ehdr.e_phoff);
-	readExactlyOrDie(fd, phdr_buffer, ehdr.e_phnum * ehdr.e_phentsize);
+	if(!phdr_buffer)
+		return LinkerError::outOfMemory;
+
+	if(!trySeek(fd, ehdr.e_phoff)) {
+		getAllocator().deallocate(phdr_buffer, ehdr.e_phnum * ehdr.e_phentsize);
+		return LinkerError::invalidProgramHeader;
+	}
+	if(!tryReadExactly(fd, phdr_buffer, ehdr.e_phnum * ehdr.e_phentsize)) {
+		getAllocator().deallocate(phdr_buffer, ehdr.e_phnum * ehdr.e_phentsize);
+		return LinkerError::invalidProgramHeader;
+	}
 
 	object->phdrPointer = phdr_buffer;
 	object->phdrCount = ehdr.e_phnum;
@@ -448,11 +473,13 @@ void ObjectRepository::_fetchFromFile(SharedObject *object, int fd) {
 	if (mlibc::sys_vm_map(nullptr,
 			highest_address - object->baseAddress, PROT_NONE,
 			MAP_PRIVATE | MAP_ANONYMOUS, -1, 0, &mappedAddr)) {
-		mlibc::panicLogger() << "sys_vm_map failed when allocating address space for DSO \""
+		mlibc::infoLogger() << "sys_vm_map failed when allocating address space for DSO \""
 				<< object->name << "\""
 				<< ", base " << (void *)object->baseAddress
 				<< ", requested " << (highest_address - object->baseAddress) << " bytes"
 				<< frg::endlog;
+		getAllocator().deallocate(phdr_buffer, ehdr.e_phnum * ehdr.e_phentsize);
+		return LinkerError::outOfMemory;
 	}
 
 	object->baseAddress = reinterpret_cast<uintptr_t>(mappedAddr);
@@ -522,9 +549,9 @@ void ObjectRepository::_fetchFromFile(SharedObject *object, int fd) {
 						MAP_PRIVATE | MAP_FIXED | MAP_ANONYMOUS, -1, 0, &map_pointer))
 					__ensure(!"sys_vm_map failed");
 
-				seekOrDie(fd, phdr->p_offset);
-				readExactlyOrDie(fd, reinterpret_cast<char *>(map_address) + misalign,
-						phdr->p_filesz);
+				__ensure(trySeek(fd, phdr->p_offset));
+				__ensure(tryReadExactly(fd, reinterpret_cast<char *>(map_address) + misalign,
+						phdr->p_filesz));
 			#endif
 			// Take care of removing superfluous permissions.
 			if(mlibc::sys_vm_protect && ((prot & PROT_WRITE) == 0))
@@ -551,6 +578,8 @@ void ObjectRepository::_fetchFromFile(SharedObject *object, int fd) {
 					<< frg::hex_fmt(phdr->p_type) << " in DSO " << object->name << frg::endlog;
 		}
 	}
+
+	return frg::success;
 }
 
 // --------------------------------------------------------
@@ -685,9 +714,9 @@ void ObjectRepository::_parseDynamic(SharedObject *object) {
 			object->preInitArraySize = dynamic->d_un.d_val;
 			break;
 		case DT_DEBUG:
-#if ELF_CLASS == 32
+#if ELF_CLASS == ELFCLASS32
 			dynamic->d_un.d_val = reinterpret_cast<Elf32_Word>(&globalDebugInterface);
-#elif ELF_CLASS == 64
+#elif ELF_CLASS == ELFCLASS64
 			dynamic->d_un.d_val = reinterpret_cast<Elf64_Xword>(&globalDebugInterface);
 #endif
 			break;
@@ -743,7 +772,7 @@ void ObjectRepository::_discoverDependencies(SharedObject *object,
 				object, localScope, false, rts);
 		if(!library)
 			mlibc::panicLogger() << "Could not satisfy dependency " << library_str << frg::endlog;
-		object->dependencies.push(library);
+		object->dependencies.push(library.value());
 	}
 }
 

options/rtdl/generic/linker.hpp

@@ -3,6 +3,7 @@
 #include <frg/optional.hpp>
 #include <frg/string.hpp>
 #include <frg/vector.hpp>
+#include <frg/expected.hpp>
 #include <mlibc/allocator.hpp>
 #include <mlibc/tcb.hpp>
 
@@ -21,6 +22,16 @@ enum class TlsModel {
 	dynamic
 };
 
+enum class LinkerError {
+	success,
+	notFound,
+	fileTooShort,
+	notElf,
+	wrongElfType,
+	outOfMemory,
+	invalidProgramHeader
+};
+
 // --------------------------------------------------------
 // ObjectRepository
 // --------------------------------------------------------
@@ -48,10 +59,10 @@ struct ObjectRepository {
 			size_t phdr_entry_size, size_t num_phdrs, void *entry_pointer,
 			uint64_t rts);
 
-	SharedObject *requestObjectWithName(frg::string_view name,
+	frg::expected<LinkerError, SharedObject *> requestObjectWithName(frg::string_view name,
 			SharedObject *origin, Scope *localScope, bool createScope, uint64_t rts);
 
-	SharedObject *requestObjectAtPath(frg::string_view path,
+	frg::expected<LinkerError, SharedObject *> requestObjectAtPath(frg::string_view path,
 			Scope *localScope, bool createScope, uint64_t rts);
 
 	SharedObject *findCaller(void *address);
@@ -65,7 +76,7 @@ struct ObjectRepository {
 	void _fetchFromPhdrs(SharedObject *object, void *phdr_pointer,
 			size_t phdr_entry_size, size_t num_phdrs, void *entry_pointer);
 
-	void _fetchFromFile(SharedObject *object, int fd);
+	frg::expected<LinkerError, void> _fetchFromFile(SharedObject *object, int fd);
 
 	void _parseDynamic(SharedObject *object);
 

options/rtdl/generic/main.cpp

@@ -167,7 +167,7 @@ extern "C" void *lazyRelocate(SharedObject *object, unsigned int rel_index) {
 	auto symbol_index = ELF_R_SYM(reloc->r_info);
 
 	__ensure(type == R_X86_64_JUMP_SLOT);
-	__ensure(ELF_CLASS == 64);
+	__ensure(ELF_CLASS == ELFCLASS64);
 
 	auto symbol = (elf_sym *)(object->baseAddress + object->symbolTableOffset
 			+ symbol_index * sizeof(elf_sym));
@@ -543,6 +543,7 @@ void *__dlapi_open(const char *file, int flags, void *returnAddress) {
 		bool isGlobal = flags & RTLD_GLOBAL;
 		Scope *newScope = isGlobal ? globalScope.get() : nullptr;
 
+		frg::expected<LinkerError, SharedObject *> objectResult;
 		if (frg::string_view{file}.find_first('/') == size_t(-1)) {
 			// In order to know which RUNPATH / RPATH to process, we must find the calling object.
 			SharedObject *origin = initialRepository->findCaller(returnAddress);
@@ -551,15 +552,37 @@ void *__dlapi_open(const char *file, int flags, void *returnAddress) {
 					<< "(ra = " << returnAddress << ")" << frg::endlog;
 			}
 
-			object = initialRepository->requestObjectWithName(file, origin, newScope, !isGlobal, rts);
+			objectResult = initialRepository->requestObjectWithName(file, origin, newScope, !isGlobal, rts);
 		} else {
-			object = initialRepository->requestObjectAtPath(file, newScope, !isGlobal, rts);
+			objectResult = initialRepository->requestObjectAtPath(file, newScope, !isGlobal, rts);
 		}
 
-		if(!object) {
-			lastError = "Cannot locate requested DSO";
+		if(!objectResult) {
+			switch (objectResult.error()) {
+			case LinkerError::success:
+				__builtin_unreachable();
+			case LinkerError::notFound:
+				lastError = "Cannot locate requested DSO";
+				break;
+			case LinkerError::fileTooShort:
+				lastError = "File too short";
+				break;
+			case LinkerError::notElf:
+				lastError = "File is not an ELF file";
+				break;
+			case LinkerError::wrongElfType:
+				lastError = "File has wrong ELF type";
+				break;
+			case LinkerError::outOfMemory:
+				lastError = "Out of memory";
+				break;
+			case LinkerError::invalidProgramHeader:
+				lastError = "File has invalid program header";
+				break;
+			}
 			return nullptr;
 		}
+		object = objectResult.value();
 
 		Loader linker{object->localScope, nullptr, false, rts};
 		linker.linkObjects(object);

options/rtdl/riscv64/elf.hpp

@@ -2,7 +2,8 @@
 
 #include <elf.h>
 
-#define ELF_CLASS 64
+#define ELF_CLASS ELFCLASS64
+#define ELF_MACHINE EM_RISCV
 
 using elf_ehdr = Elf64_Ehdr;
 using elf_phdr = Elf64_Phdr;

options/rtdl/x86/elf.hpp

@@ -2,7 +2,8 @@
 
 #include <elf.h>
 
-#define ELF_CLASS 32
+#define ELF_CLASS ELFCLASS32
+#define ELF_MACHINE EM_386
 
 using elf_ehdr = Elf32_Ehdr;
 using elf_phdr = Elf32_Phdr;

options/rtdl/x86_64/elf.hpp

@@ -2,7 +2,8 @@
 
 #include <elf.h>
 
-#define ELF_CLASS 64
+#define ELF_CLASS ELFCLASS64
+#define ELF_MACHINE EM_X86_64
 
 using elf_ehdr = Elf64_Ehdr;
 using elf_phdr = Elf64_Phdr;