diff --git a/include/slotsboxmalloc/slotsboxobj.h b/include/slotsboxmalloc/slotsboxobj.h index d99d608..d6183b8 100644 --- a/include/slotsboxmalloc/slotsboxobj.h +++ b/include/slotsboxmalloc/slotsboxobj.h @@ -54,7 +54,8 @@ typedef struct { atomic_int lock; uint32_t hot_id1[3]; uint64_t hot_base[3]; - uint8_t _pad[24]; + uint8_t no_reclaimable; + uint8_t _pad[23]; } sbo_lock_t; _Static_assert(ATOMIC_INT_LOCK_FREE == 2, "atomic_int must be lock-free"); _Static_assert(sizeof(sbo_lock_t) == 64, "sbo_lock_t must be 64 bytes"); @@ -188,7 +189,7 @@ static inline sbo_ctx_t sbo_ctx_for(sbo_meta_t *meta, uint8_t si, sbo_lock_t *lk return c; } static inline sbo_box_t *sbo_box_at(const sbo_ctx_t *c, int32_t id) { - return (sbo_box_t *)(c->pool + (uint32_t)id * c->stride + c->head); + return (sbo_box_t *)(c->pool + (size_t)(uint32_t)id * c->stride + c->head); } static inline uint8_t sbo_mask_contmax(uint64_t m) { @@ -353,6 +354,8 @@ static inline void sbo_free_obj_slots(const sbo_ctx_t *c, sbo_box_t *box, uint8_ if (start + len > SBO_N) return; box->states[start] = SBO_FREE; box->free_mask |= (((1ULL << len) - 1ULL) << start); + if (box->parent >= 0 && box->free_mask == ~0ULL && box->child_mask == 0) + c->lk->no_reclaimable = 0; sbo_after_mask_change(c, box); } @@ -390,11 +393,11 @@ static uint64_t sbo_find_alloc(sbo_ctx_t *c, sbo_box_t *box, } uint32_t need_blk = (uint32_t)(box->objlevel - usage.level); - if (!box->free_mask || c->max_blocks - (uint32_t)c->pool_meta->used_blocks < need_blk) { - sbo_reclaim_empty(c, sbo_box_at(c, 0)); - if (!box->free_mask || c->max_blocks - (uint32_t)c->pool_meta->used_blocks < need_blk) - return SBO_ALLOC_FAIL; - } + /* Never reclaim from the root here: it can free `box` (or an ancestor) while + * we are still inside it. sbo_alloc reclaims at top level and retries. */ + if (!box->free_mask && box->child_mask) sbo_reclaim_empty(c, box); + if (!box->free_mask || c->max_blocks - (uint32_t)c->pool_meta->used_blocks < need_blk) + return SBO_ALLOC_FAIL; int64_t cid = blocks_alloc(c->pool_meta, c->pool); if (cid < 0) return SBO_ALLOC_FAIL; @@ -641,11 +644,20 @@ uint64_t sbo_alloc(void *metaptr, size_t size) { sbo_ctx_t c = sbo_ctx_for(meta, (uint8_t)si, lk); sbo_box_t *root = sbo_box_at(&c, 0); uint64_t off = SBO_ALLOC_FAIL; - if (sbo_combined(root) < need_enc && root->child_mask) + if (sbo_combined(root) < need_enc && root->child_mask && !lk->no_reclaimable) { sbo_reclaim_empty(&c, root); + lk->no_reclaimable = 1; + } if (sbo_combined(root) >= need_enc) { if (!sbo_try_hint(&c, root, usage, need_enc, &off)) { off = sbo_find_alloc(&c, root, usage, need_enc); + if (off == SBO_ALLOC_FAIL && root->child_mask && !lk->no_reclaimable) { + uint64_t used = c.pool_meta->used_blocks; + sbo_reclaim_empty(&c, root); + lk->no_reclaimable = 1; + if (c.pool_meta->used_blocks < used) + off = sbo_find_alloc(&c, root, usage, need_enc); + } if (off != SBO_ALLOC_FAIL && usage.level >= 1 && usage.level <= 3 && c.last_id >= 0) { uint32_t bsh = (uint32_t)usage.level * SBO_N_BITS + 3; diff --git a/tutorial/18_model_check.c b/tutorial/18_model_check.c index 92b60c4..ee9e8b8 100644 --- a/tutorial/18_model_check.c +++ b/tutorial/18_model_check.c @@ -7,6 +7,7 @@ #include #include #include +#include #include #define META_SIZE (16 * 1024 * 1024) @@ -241,6 +242,172 @@ static void phase_single_shard_mt(int nthr, int iters) { printf("F single-shard MT done (errors so far=%ld)\n", (long)g_err); } +/* Single-shard arena whose box pool holds exactly `blocks` boxes (root included). */ +static uint8_t *tiny_pool_arena(size_t data, uint32_t blocks) { + size_t probe = sbo_meta_size(data, 64 * 1024); + uint8_t *p = calloc(1, probe); + if (!p || sbo_init(p, probe, data) != 0) { free(p); return NULL; } + size_t stride = ((sbo_meta_t *)p)->block_stride; + free(p); + size_t ms = sbo_meta_size(data, blocks * stride); + uint8_t *m = calloc(1, ms); + if (!m || sbo_init(m, ms, data) != 0 || ((sbo_meta_t *)m)->max_blocks != blocks) { + free(m); + return NULL; + } + return m; +} + +static bool overlaps(uint64_t a, uint64_t as, uint64_t b, uint64_t bs) { + return a < b + bs && b < a + as; +} + +/* With every object freed, root + (blocks-1) boxes must be rebuildable: each + * 63*512B object needs its own L2 box. Fails if a box block leaked. */ +static bool tiny_pool_recoverable(uint8_t *m, uint32_t blocks) { + uint64_t o[64]; + uint32_t n = 0; + bool ok = true; + for (; n < blocks - 1 && n < 64; n++) { + o[n] = sbo_alloc(m, 63 * 512); + if (o[n] == FAIL64) { ok = false; break; } + } + for (uint32_t i = 0; i < n; i++) sbo_free(m, o[i]); + return ok; +} + +/* Pool exhausted while the search sits in an empty child box: that box must + * not be reclaimed under the in-flight allocation (it used to be, and the new + * object landed in a detached box overlapping later allocations). */ +static void phase_tiny_pool(void) { + const size_t data = 2097152; + uint8_t *m = tiny_pool_arena(data, 3); + if (!m) { printf("tiny pool: init fail\n"); g_err++; return; } + uint64_t a = sbo_alloc(m, 63 * 512), b = sbo_alloc(m, 63 * 512); + if (a == FAIL64 || b == FAIL64) { printf("tiny pool: setup alloc fail\n"); g_err++; free(m); return; } + sbo_free(m, b); + uint64_t o[3] = {a, sbo_alloc(m, 8), sbo_alloc(m, 63 * 32768)}; + const size_t req[3] = {63 * 512, 8, 63 * 32768}; + for (int i = 0; i < 3; i++) { + if (o[i] == FAIL64) { printf("tiny pool: alloc %zu failed\n", req[i]); g_err++; continue; } + if (sbo_allocated_size(m, o[i]) != expect_cap(req[i])) g_err++; + for (int j = 0; j < i; j++) + if (o[j] != FAIL64 && overlaps(o[i], expect_cap(req[i]), o[j], expect_cap(req[j]))) { + printf("tiny pool: OVERLAP off=%lu and off=%lu\n", (unsigned long)o[i], (unsigned long)o[j]); + g_err++; + } + } + for (int i = 0; i < 3; i++) if (o[i] != FAIL64) sbo_free(m, o[i]); + if (!tiny_pool_recoverable(m, 3)) { printf("tiny pool: box leaked\n"); g_err++; } + free(m); + printf("G tiny pool reclaim done (errors so far=%ld)\n", (long)g_err); +} + +/* Randomized churn on a box pool small enough to be exhausted constantly. */ +static void phase_tiny_pool_churn(int iters) { + static const size_t szs[] = {8, 100, 512, 4096, 32256, 65536, 300000}; + const size_t data = 2097152; + const uint32_t blocks = 5; + uint8_t *m = tiny_pool_arena(data, blocks); + if (!m) { printf("tiny churn: init fail\n"); g_err++; return; } + uint64_t rs = 0xC0FFEEULL, off[64], cap[64]; + int nl = 0; + long ok = 0; + for (int i = 0; i < iters; i++) { + if (nl > 0 && (nl == 64 || (rng(&rs) & 1))) { + int k = (int)(rng(&rs) % (uint64_t)nl); + if (sbo_allocated_size(m, off[k]) != cap[k]) g_err++; + sbo_free(m, off[k]); + off[k] = off[--nl]; cap[k] = cap[nl]; + continue; + } + size_t req = szs[rng(&rs) % (sizeof(szs) / sizeof(szs[0]))]; + uint64_t o = sbo_alloc(m, req); + if (o == FAIL64) continue; + ok++; + uint64_t c = expect_cap(req); + if (sbo_allocated_size(m, o) != c || o + c > data) { g_err++; sbo_free(m, o); continue; } + for (int j = 0; j < nl; j++) + if (overlaps(o, c, off[j], cap[j])) { + printf("tiny churn: OVERLAP iter=%d off=%lu cap=%lu vs off=%lu cap=%lu\n", i, + (unsigned long)o, (unsigned long)c, (unsigned long)off[j], (unsigned long)cap[j]); + g_err++; + break; + } + off[nl] = o; cap[nl++] = c; + } + while (nl > 0) { nl--; sbo_free(m, off[nl]); } + if (ok == 0) { printf("tiny churn: no successful allocations\n"); g_err++; } + if (!tiny_pool_recoverable(m, blocks)) { printf("tiny churn: box leaked\n"); g_err++; } + free(m); + printf("H tiny pool churn done (errors so far=%ld)\n", (long)g_err); +} + +/* Full-arena repeated failing allocs must short-circuit via no_reclaimable + * without O(N) full-tree scans, while freeing a box re-enables lazy reclaim. */ +static void phase_full_arena_oom(void) { + const size_t data = 2097152; /* 2MB = 64 L2 slots; fill all 64 L2 boxes with 504B (63*8B) objects */ + size_t ms = sbo_meta_size(data, 2 * 1024 * 1024); + uint8_t *m = calloc(1, ms); + uint64_t *offs = malloc(4096 * sizeof(uint64_t)); + if (!m || !offs || sbo_init(m, ms, data) != 0) { + printf("full oom: init fail\n"); + g_err++; free(m); free(offs); return; + } + /* Each L1 box holds one 504B object (slots 1..63) + one 8B object (slot 0) -> 64/64 slots used! + * And root L2 has 64 child L1 boxes -> 4096 L1 boxes total! */ + size_t n504 = 0, n8 = 0; + uint64_t *offs8 = malloc(4096 * sizeof(uint64_t)); + if (!offs8) { g_err++; free(m); free(offs); return; } + while (n504 < 4096) { + uint64_t o = sbo_alloc(m, 504); + if (o == FAIL64) break; + offs[n504++] = o; + } + while (n8 < 4096) { + uint64_t o = sbo_alloc(m, 8); + if (o == FAIL64) break; + offs8[n8++] = o; + } + if (n504 != 4096 || n8 != 4096) { + printf("full oom: expected 4096x504 + 4096x8, got %zu + %zu\n", n504, n8); + g_err++; + } + if (sbo_alloc(m, 8) != FAIL64) g_err++; /* sets no_reclaimable = 1 */ + if (sbo_alloc(m, 512) != FAIL64) g_err++; + + struct timespec t0, t1; + clock_gettime(CLOCK_MONOTONIC, &t0); + for (int i = 0; i < 10000; i++) { + if (sbo_alloc(m, 8) != FAIL64) { g_err++; break; } + if (sbo_alloc(m, 32768) != FAIL64) { g_err++; break; } + } + clock_gettime(CLOCK_MONOTONIC, &t1); + double elapsed_ms = (double)(t1.tv_sec - t0.tv_sec) * 1e3 + + (double)(t1.tv_nsec - t0.tv_nsec) * 1e-6; + if (elapsed_ms > 500.0) { + printf("full oom: 20000 failing allocs took %.2f ms (no_reclaimable broken?)\n", elapsed_ms); + g_err++; + } + + /* Free one entire L1 box [0, 512) -> clears no_reclaimable -> 512B L2 slot alloc succeeds */ + for (size_t i = 0; i < n504; i++) + if (offs[i] < 512) sbo_free(m, offs[i]); + for (size_t i = 0; i < n8; i++) + if (offs8[i] < 512) sbo_free(m, offs8[i]); + uint64_t l2_slot = sbo_alloc(m, 512); + if (l2_slot == FAIL64) { printf("full oom: reclaim after free failed\n"); g_err++; } + else sbo_free(m, l2_slot); + for (size_t i = 0; i < n504; i++) + if (offs[i] >= 512) sbo_free(m, offs[i]); + for (size_t i = 0; i < n8; i++) + if (offs8[i] >= 512) sbo_free(m, offs8[i]); + free(offs8); + free(offs); + free(m); + printf("I full arena OOM fast-fail done (%.2f ms, errors so far=%ld)\n", elapsed_ms, (long)g_err); +} + int main(int argc, char **argv) { if (argc > 3) { printf("usage: %s [threads 1..254] [iters>=1]\n", argv[0]); return 2; } int nthr = argc > 1 ? atoi(argv[1]) : 4; @@ -259,6 +426,9 @@ int main(int argc, char **argv) { phase_edges(); phase_small_arenas(); phase_single_shard_mt(nthr, iters); + phase_tiny_pool(); + phase_tiny_pool_churn(iters); + phase_full_arena_oom(); printf("18_model_check: %s\n", g_err == 0 ? "PASS" : "FAIL"); free(g_meta); free(g_data); free(g_claim); return g_err == 0 ? 0 : 1;