New allocation algorithm (#1166)

Replaced algorithm in class arena with one that uses mmap()'d address
spaces and demand paging for each semispace and initializes the current
semispace of each arena at start time to avoid and the second semispace
of each arena at swap time to avoid relying on Undefined Behavior or
having an extra test in kore_arena_alloc().

---------

Co-authored-by: Dwight Guth <[email protected]>

include/runtime/arena.h

@@ -1,38 +1,49 @@
 #ifndef ARENA_H
 #define ARENA_H
 
+#include <algorithm>
 #include <cstddef>
+#include <cstdint>
 #include <sys/types.h>
+#include <utility>
 
 #include "runtime/alloc.h"
 
 extern "C" {
 
+size_t const HYPERBLOCK_SIZE = (size_t)BLOCK_SIZE * 1024 * 1024;
+
 // An arena can be used to allocate objects that can then be deallocated all at
 // once.
 class arena {
 public:
   arena(char id)
-      : allocation_semispace_id(id) { }
+      : allocation_semispace_id(id) {
+    initialize_semispace();
+  }
 
   // Allocates the requested number of bytes as a contiguous region and returns a
   // pointer to the first allocated byte.
-  // If called with requested size greater than the maximun single allocation
-  // size, the space is allocated in a general (not garbage collected pool).
   void *kore_arena_alloc(size_t requested);
 
   // Returns the address of the first byte that belongs in the given arena.
   // Returns 0 if nothing has been allocated ever in that arena.
-  char *arena_start_ptr() const;
+  char *arena_start_ptr() const {
+    return current_addr_ptr ? current_addr_ptr + sizeof(memory_block_header)
+                            : nullptr;
+  }
 
   // Returns a pointer to a location holding the address of last allocated
   // byte in the given arena plus 1.
   // This address is 0 if nothing has been allocated ever in that arena.
-  char **arena_end_ptr();
+  char **arena_end_ptr() { return &allocation_ptr; }
 
   // return the total number of allocatable bytes currently in the arena in its
   // active semispace.
-  size_t arena_size() const;
+  size_t arena_size() const {
+    update_num_blocks();
+    return BLOCK_SIZE * std::max(num_blocks, num_collection_blocks);
+  }
 
   // Clears the current allocation space by setting its start back to its first
   // block. It is used during garbage collection to effectively collect all of the
@@ -41,15 +52,18 @@ class arena {
 
   // Resizes the last allocation as long as the resize does not require a new
   // block allocation.
-  // Returns the address of the byte following the last newlly allocated byte when
-  // the resize succeeds, returns 0 otherwise.
-  void *arena_resize_last_alloc(ssize_t increase);
+  // Returns the address of the byte following the last newlly allocated byte.
+  void *arena_resize_last_alloc(ssize_t increase) {
+    return (allocation_ptr += increase);
+  }
 
   // Returns the given arena's current collection semispace ID.
   // Each arena has 2 semispace IDs one equal to the arena ID and the other equal
   // to the 1's complement of the arena ID. At any time one of these semispaces
   // is used for allocation and the other is used for collection.
-  char get_arena_collection_semispace_id() const;
+  char get_arena_collection_semispace_id() const {
+    return ~allocation_semispace_id;
+  }
 
   // Exchanges the current allocation and collection semispaces and clears the new
   // current allocation semispace by setting its start back to its first block.
@@ -61,7 +75,7 @@ class arena {
   // by the blocks of that arena. This difference will include blocks containing
   // sentinel bytes. Undefined behavior will result if the pointers belong to
   // different arenas.
-  static ssize_t ptr_diff(char *ptr1, char *ptr2);
+  static ssize_t ptr_diff(char *ptr1, char *ptr2) { return ptr1 - ptr2; }
 
   // Given a starting pointer to an address allocated in an arena and a size in
   // bytes, this function returns a pointer to an address allocated in the
@@ -72,42 +86,70 @@ class arena {
   // 3rd argument: the address of last allocated byte in the arena plus 1
   // Return value: the address allocated in the arena after size bytes from the
   //               starting pointer, or 0 if this is equal to the 3rd argument.
-  static char *move_ptr(char *ptr, size_t size, char const *arena_end_ptr);
+  static char *move_ptr(char *ptr, size_t size, char const *arena_end_ptr) {
+    char *next_ptr = ptr + size;
+    return (next_ptr == arena_end_ptr) ? 0 : next_ptr;
+  }
 
   // Returns the ID of the semispace where the given address was allocated.
   // The behavior is undefined if called with an address that has not been
   // allocated within an arena.
   static char get_arena_semispace_id_of_object(void *ptr);
 
 private:
-  struct memory_block_header {
-    char *next_block;
+  union memory_block_header {
+    //
+    //	Currently the header just holds the semispace id. But we need it to be a
+    //	multiple of sizeof(char*) for alignment purposes so we add a dummy char*.
+    //
     char semispace;
+    char *alignment_dummy;
   };
 
-  void fresh_block();
-  static memory_block_header *mem_block_header(void *ptr);
+  //
+  //	We update the number of 1MB blocks actually written to, only when we need this value,
+  //	or before a garbage collection rather than trying to determine when we write to a fresh block.
+  //
+  void update_num_blocks() const {
+    //
+    //	Calculate how many 1M blocks of the current arena we used.
+    //
+    size_t num_used_blocks
+        = (allocation_ptr - current_addr_ptr - 1) / BLOCK_SIZE + 1;
+    if (num_used_blocks > num_blocks)
+      num_blocks = num_used_blocks;
+  }
+
+  void initialize_semispace();
 
-  // helper function for `kore_arena_alloc`. Do not call directly.
-  void *do_alloc_slow(size_t requested);
+  static memory_block_header *mem_block_header(void *ptr) {
+    uintptr_t address = reinterpret_cast<uintptr_t>(ptr);
+    return reinterpret_cast<arena::memory_block_header *>(
+        (address - 1) & ~(HYPERBLOCK_SIZE - 1));
+  }
 
-  char *first_block; // beginning of first block
-  char *block; // where allocations are being made in current block
-  char *block_start; // start of current block
-  char *block_end; // 1 past end of current block
-  char *first_collection_block; // beginning of other semispace
-  size_t num_blocks; // number of blocks in current semispace
-  size_t num_collection_blocks; // number of blocks in other semispace
+  //
+  //	Current semispace where allocations are being made.
+  //
+  char *current_addr_ptr; // pointer to start of current address space
+  char *allocation_ptr; // next available location in current semispace
+  char *tripwire; // allocating past this triggers slow allocation
+  mutable size_t
+      num_blocks; // notional number of BLOCK_SIZE blocks in current semispace
   char allocation_semispace_id; // id of current semispace
+  //
+  //	Semispace where allocations will be made during and after garbage collect.
+  //
+  char *collection_addr_ptr
+      = nullptr; // pointer to start of collection address space
+  size_t num_collection_blocks
+      = 0; // notional number of BLOCK_SIZE blocks in collection semispace
 };
 
 // Macro to define a new arena with the given ID. Supports IDs ranging from 0 to
 // 127.
 #define REGISTER_ARENA(name, id) static thread_local arena name(id)
 
-#define MEM_BLOCK_START(ptr)                                                   \
-  ((char *)(((uintptr_t)(ptr)-1) & ~(BLOCK_SIZE - 1)))
-
 #ifdef __MACH__
 //
 //	thread_local disabled for Apple
@@ -120,16 +162,51 @@ extern thread_local bool time_for_collection;
 size_t get_gc_threshold();
 
 inline void *arena::kore_arena_alloc(size_t requested) {
-  if (block + requested > block_end) {
-    return do_alloc_slow(requested);
+  if (allocation_ptr + requested >= tripwire) {
+    //
+    //	We got close to or past the last location accessed in this address range so far,
+    //	depending on the requested size and tripwire setting. This triggers a garbage
+    //	collect when allowed.
+    //
+    time_for_collection = true;
+    tripwire = current_addr_ptr
+               + HYPERBLOCK_SIZE; // won't trigger again until arena swap
   }
-  void *result = block;
-  block += requested;
+  void *result = allocation_ptr;
+  allocation_ptr += requested;
   MEM_LOG(
-      "Allocation at %p (size %zd), next alloc at %p (if it fits)\n", result,
-      requested, block);
+      "Allocation at %p (size %zd), next alloc at %p\n", result, requested,
+      block);
   return result;
 }
+
+inline void arena::arena_clear() {
+  //
+  //	We set the allocation pointer to the first available address.
+  //
+  allocation_ptr = arena_start_ptr();
+  //
+  //	If the number of blocks we've touched is >= threshold, we want to trigger
+  //	a garbage collection if we get within 1 block of the end of this area.
+  //	Otherwise we only want to generate a garbage collect if we allocate off the
+  //	end of this area.
+  //
+  tripwire = current_addr_ptr
+             + (num_blocks - (num_blocks >= get_gc_threshold())) * BLOCK_SIZE;
 }
 
+inline void arena::arena_swap_and_clear() {
+  update_num_blocks(); // so we save the correct number of touched blocks
+  std::swap(current_addr_ptr, collection_addr_ptr);
+  std::swap(num_blocks, num_collection_blocks);
+  allocation_semispace_id = ~allocation_semispace_id;
+  if (current_addr_ptr == nullptr) {
+    //
+    //	The other semispace hasn't be initialized yet.
+    //
+    initialize_semispace();
+  } else
+    arena_clear();
+}
+}
 #endif // ARENA_H