Move heap ownership info from chunk to pagemap

.release-notes/4368.md

@@ -0,0 +1,3 @@
+## Move heap ownership info from chunk to pagemap
+
+An internal runtime change has been made around where/how heap chunk ownership information is stored in order to improve performance. The tradeoff is that this will now use a little more memory in order to realize the performance gains.

packages/builtin_test/_test.pony

@@ -11,7 +11,7 @@ use @pony_alloc_final[Pointer[U8]](ctx: Pointer[None], size: USize)
 use @pony_exitcode[None](code: I32)
 use @pony_get_exitcode[I32]()
 use @pony_triggergc[None](ctx: Pointer[None])
-use @ponyint_pagemap_get[Pointer[None]](p: Pointer[None] tag)
+use @ponyint_pagemap_get_chunk[Pointer[None]](p: Pointer[None] tag)
 
 use "pony_test"
 use "collections"
@@ -617,12 +617,12 @@ class \nodoc\ iso _TestStringTrimInPlace is UnitTest
     space: USize = 0)
   =>
     let copy: String ref = orig.clone()
-    let pre_trim_pagemap = @ponyint_pagemap_get(copy.cpointer())
+    let pre_trim_pagemap = @ponyint_pagemap_get_chunk(copy.cpointer())
     copy.trim_in_place(from, to)
     h.assert_eq[String box](expected, copy)
     h.assert_eq[USize](space, copy.space())
     h.assert_eq[String box](expected, copy.clone()) // safe to clone
-    let post_trim_pagemap = @ponyint_pagemap_get(copy.cpointer())
+    let post_trim_pagemap = @ponyint_pagemap_get_chunk(copy.cpointer())
     if copy.space() == 0 then
       h.assert_eq[USize](0, post_trim_pagemap.usize())
     else
@@ -1480,11 +1480,11 @@ class \nodoc\ iso _TestArrayTrimInPlace is UnitTest
     space: USize = 0)
   =>
     let copy: Array[U8] ref = orig.clone()
-    let pre_trim_pagemap = @ponyint_pagemap_get(copy.cpointer())
+    let pre_trim_pagemap = @ponyint_pagemap_get_chunk(copy.cpointer())
     copy.trim_in_place(from, to)
     h.assert_eq[USize](space, copy.space())
     h.assert_array_eq[U8](expected, copy)
-    let post_trim_pagemap = @ponyint_pagemap_get(copy.cpointer())
+    let post_trim_pagemap = @ponyint_pagemap_get_chunk(copy.cpointer())
     if copy.space() == 0 then
       h.assert_eq[USize](0, post_trim_pagemap.usize())
     else

src/libponyrt/gc/gc.c

@@ -482,7 +482,8 @@ static void acq_or_rel_remote_object(pony_ctx_t* ctx, pony_actor_t* actor,
 void ponyint_gc_sendobject(pony_ctx_t* ctx, void* p, pony_type_t* t,
   int mutability)
 {
-  chunk_t* chunk = ponyint_pagemap_get(p);
+  pony_actor_t* actor = NULL;
+  chunk_t* chunk = ponyint_pagemap_get(p, &actor);
 
   // Don't gc memory that wasn't pony_allocated, but do recurse.
   if(chunk == NULL)
@@ -492,8 +493,6 @@ void ponyint_gc_sendobject(pony_ctx_t* ctx, void* p, pony_type_t* t,
     return;
   }
 
-  pony_actor_t* actor = ponyint_heap_owner(chunk);
-
   if(actor == ctx->current)
     send_local_object(ctx, p, t, mutability);
   else
@@ -503,7 +502,8 @@ void ponyint_gc_sendobject(pony_ctx_t* ctx, void* p, pony_type_t* t,
 void ponyint_gc_recvobject(pony_ctx_t* ctx, void* p, pony_type_t* t,
   int mutability)
 {
-  chunk_t* chunk = ponyint_pagemap_get(p);
+  pony_actor_t* actor = NULL;
+  chunk_t* chunk = ponyint_pagemap_get(p, &actor);
 
   // Don't gc memory that wasn't pony_allocated, but do recurse.
   if(chunk == NULL)
@@ -513,8 +513,6 @@ void ponyint_gc_recvobject(pony_ctx_t* ctx, void* p, pony_type_t* t,
     return;
   }
 
-  pony_actor_t* actor = ponyint_heap_owner(chunk);
-
   if(actor == ctx->current)
     recv_local_object(ctx, p, t, mutability);
   else
@@ -524,7 +522,8 @@ void ponyint_gc_recvobject(pony_ctx_t* ctx, void* p, pony_type_t* t,
 void ponyint_gc_markobject(pony_ctx_t* ctx, void* p, pony_type_t* t,
   int mutability)
 {
-  chunk_t* chunk = ponyint_pagemap_get(p);
+  pony_actor_t* actor = NULL;
+  chunk_t* chunk = ponyint_pagemap_get(p, &actor);
 
   // Don't gc memory that wasn't pony_allocated, but do recurse.
   if(chunk == NULL)
@@ -534,8 +533,6 @@ void ponyint_gc_markobject(pony_ctx_t* ctx, void* p, pony_type_t* t,
     return;
   }
 
-  pony_actor_t* actor = ponyint_heap_owner(chunk);
-
   if(actor == ctx->current)
     mark_local_object(ctx, chunk, p, t, mutability);
   else
@@ -545,7 +542,8 @@ void ponyint_gc_markobject(pony_ctx_t* ctx, void* p, pony_type_t* t,
 void ponyint_gc_acquireobject(pony_ctx_t* ctx, void* p, pony_type_t* t,
   int mutability)
 {
-  chunk_t* chunk = ponyint_pagemap_get(p);
+  pony_actor_t* actor = NULL;
+  chunk_t* chunk = ponyint_pagemap_get(p, &actor);
 
   // Don't gc memory that wasn't pony_allocated, but do recurse.
   if(chunk == NULL)
@@ -555,8 +553,6 @@ void ponyint_gc_acquireobject(pony_ctx_t* ctx, void* p, pony_type_t* t,
     return;
   }
 
-  pony_actor_t* actor = ponyint_heap_owner(chunk);
-
   if(actor == ctx->current)
     acquire_local_object(ctx, p, t, mutability);
   else
@@ -566,7 +562,8 @@ void ponyint_gc_acquireobject(pony_ctx_t* ctx, void* p, pony_type_t* t,
 void ponyint_gc_releaseobject(pony_ctx_t* ctx, void* p, pony_type_t* t,
   int mutability)
 {
-  chunk_t* chunk = ponyint_pagemap_get(p);
+  pony_actor_t* actor = NULL;
+  chunk_t* chunk = ponyint_pagemap_get(p, &actor);
 
   // Don't gc memory that wasn't pony_allocated, but do recurse.
   if(chunk == NULL)
@@ -576,8 +573,6 @@ void ponyint_gc_releaseobject(pony_ctx_t* ctx, void* p, pony_type_t* t,
     return;
   }
 
-  pony_actor_t* actor = ponyint_heap_owner(chunk);
-
   if(actor == ctx->current)
     release_local_object(ctx, p, t, mutability);
   else
@@ -665,7 +660,7 @@ void ponyint_gc_markimmutable(pony_ctx_t* ctx, gc_t* gc)
     {
       // Mark in our heap and recurse if it wasn't already marked.
       void* p = obj->address;
-      chunk_t* chunk = ponyint_pagemap_get(p);
+      chunk_t* chunk = ponyint_pagemap_get_chunk(p);
       mark_local_object(ctx, chunk, p, obj->type, PONY_TRACE_IMMUTABLE);
     }
   }

src/libponyrt/gc/objectmap.c

@@ -71,7 +71,7 @@ void ponyint_objectmap_sweep(objectmap_t* map)
 
     if(obj->rc > 0)
     {
-      chunk_t* chunk = ponyint_pagemap_get(p);
+      chunk_t* chunk = ponyint_pagemap_get_chunk(p);
       ponyint_heap_mark_shallow(chunk, p);
     } else {
       ponyint_objectmap_clearindex(map, i);

src/libponyrt/mem/heap.c

@@ -10,9 +10,6 @@
 
 typedef struct chunk_t
 {
-  // immutable
-  pony_actor_t* actor;
-
   // used for pointer tagging
   // bit 0 (lowest bit) for keeping track of chunk type (1 = small; 0 = large)
   // bit 1 for keeping track of chunks to be cleared
@@ -238,9 +235,9 @@ static char* get_m(chunk_t* chunk)
   return (char*)((uintptr_t)chunk->m & CHUNK_M_BITMASK);
 }
 
-static void large_pagemap(char* m, size_t size, chunk_t* chunk)
+static void large_pagemap(char* m, size_t size, chunk_t* chunk, pony_actor_t* actor)
 {
-  ponyint_pagemap_set_bulk(m, chunk, size);
+  ponyint_pagemap_set_bulk(m, chunk, actor, size);
 }
 
 static void maybe_clear_chunk(chunk_t* chunk)
@@ -326,7 +323,7 @@ static void destroy_small(small_chunk_t* chunk, uint32_t mark)
   final_small(chunk, FORCE_ALL_FINALISERS);
 
   char* m = get_m((chunk_t*)chunk);
-  ponyint_pagemap_set(m, NULL);
+  ponyint_pagemap_set(m, NULL, NULL);
   POOL_FREE(block_t, m);
   POOL_FREE(small_chunk_t, chunk);
 }
@@ -340,7 +337,7 @@ static void destroy_large(large_chunk_t* chunk, uint32_t mark)
   final_large(chunk, mark);
 
   char* m = get_m((chunk_t*)chunk);
-  large_pagemap(m, chunk->size, NULL);
+  large_pagemap(m, chunk->size, NULL, NULL);
 
   if(m != NULL)
     ponyint_pool_free_size(chunk->size, m);
@@ -597,7 +594,6 @@ void* ponyint_heap_alloc_small(pony_actor_t* actor, heap_t* heap,
     }
   } else {
     small_chunk_t* n = (small_chunk_t*) POOL_ALLOC(small_chunk_t);
-    n->base.actor = actor;
     n->base.m = (char*) POOL_ALLOC(block_t);
     set_small_chunk_size(n, sizeclass);
 #ifdef USE_RUNTIMESTATS
@@ -617,7 +613,7 @@ void* ponyint_heap_alloc_small(pony_actor_t* actor, heap_t* heap,
 
     set_chunk_needs_clearing((chunk_t*)n);
 
-    ponyint_pagemap_set(get_m((chunk_t*)n), (chunk_t*)n);
+    ponyint_pagemap_set(get_m((chunk_t*)n), (chunk_t*)n, actor);
 
     heap->small_free[sizeclass] = n;
     chunk = n;
@@ -648,7 +644,6 @@ void* ponyint_heap_alloc_large(pony_actor_t* actor, heap_t* heap, size_t size,
   size = ponyint_pool_adjust_size(size);
 
   large_chunk_t* chunk = (large_chunk_t*) POOL_ALLOC(large_chunk_t);
-  chunk->base.actor = actor;
   chunk->size = size;
   chunk->base.m = (char*) ponyint_pool_alloc_size(size);
 #ifdef USE_RUNTIMESTATS
@@ -666,7 +661,7 @@ void* ponyint_heap_alloc_large(pony_actor_t* actor, heap_t* heap, size_t size,
   // note if a finaliser needs to run or not
   set_large_chunk_finaliser(chunk, (track_finalisers_mask & 1));
 
-  large_pagemap(get_m((chunk_t*)chunk), size, (chunk_t*)chunk);
+  large_pagemap(get_m((chunk_t*)chunk), size, (chunk_t*)chunk, actor);
 
   chunk->next = heap->large;
   heap->large = chunk;
@@ -685,7 +680,7 @@ void* ponyint_heap_realloc(pony_actor_t* actor, heap_t* heap, void* p,
   actor->actorstats.heap_realloc_counter++;
 #endif
 
-  chunk_t* chunk = ponyint_pagemap_get(p);
+  chunk_t* chunk = ponyint_pagemap_get_chunk(p);
 
   // We can't realloc memory that wasn't pony_alloc'ed since we can't know how
   // much to copy from the previous location.
@@ -908,18 +903,6 @@ void ponyint_heap_endgc(heap_t* heap
     heap->next_gc = heap_initialgc;
 }
 
-pony_actor_t* ponyint_heap_owner(chunk_t* chunk)
-{
-  // FIX: false sharing
-  // reading from something that will never be written
-  // but is on a cache line that will often be written
-  // called during tracing
-  // actual chunk only needed for GC tracing
-  // all other tracing only needs the owner
-  // so the owner needs the chunk and everyone else just needs the owner
-  return chunk->actor;
-}
-
 size_t ponyint_heap_size(chunk_t* chunk)
 {
   if(get_chunk_is_small_chunk(chunk))

src/libponyrt/mem/heap.h

@@ -102,8 +102,6 @@ void ponyint_heap_endgc(heap_t* heap
   );
 #endif
 
-pony_actor_t* ponyint_heap_owner(chunk_t* chunk);
-
 size_t ponyint_heap_size(chunk_t* chunk);
 
 #ifdef USE_RUNTIMESTATS