From 976d555c2753a715305abeec673ba3d90749fe44 Mon Sep 17 00:00:00 2001 From: Daniel Byrne Date: Wed, 9 Sep 2026 08:55:00 -0700 Subject: [PATCH 1/5] Add explicit asynchronous submit/poll offload API libdto so far only accelerated memcpy/memmove/memset/memcmp transparently via LD_PRELOAD. This adds a public dto.h with an explicit submit/poll API so applications can overlap DSA data movement with their own CPU work: dto_submit_memcpy / dto_submit_memset / dto_submit_memcpy_crc / dto_submit_crc -> dto_async_poll -> dto_async_crc_val dto_batch_op_new / dto_submit_batch_copy / dto_batch_poll (up to 64 copies) The operation state is caller-allocated (opaque dto_async_op) so it can be polled from any thread. Every submit takes a flags argument: DTO_SUBMIT_CC directs output toward the CPU cache when the device supports it, and DTO_SUBMIT_BOF makes the device block on page faults instead of aborting (requires a work queue configured with block-on-fault). Without BOF a faulting operation surfaces as DTO_ASYNC_FAILED and the caller redoes it on the CPU; submission returns DTO_ASYNC_FALLBACK when DSA is unavailable or the request is out of range, with nothing copied. CRC results follow the raw CRC32C convention (seed 0, no final inversion), matching _mm_crc32. The memset entry point takes (dest, c, n) like the C library, replacing the earlier page-oriented helper so the offload API is uniform across operations. Adds seven functional tests (copy, memset, crc, copy+crc, no-BOF fault contract, argument validation, batch) that pass both on DSA hardware and on the CPU-fallback path. --- CMakeLists.txt | 4 + Makefile | 1 + dto.c | 278 +++++++++++++++++++++++++++++++++++++++- dto.h | 139 ++++++++++++++++++++ tests/test_functional.c | 232 +++++++++++++++++++++++++++++++++ 5 files changed, 650 insertions(+), 4 deletions(-) create mode 100644 dto.h diff --git a/CMakeLists.txt b/CMakeLists.txt index f63146a..102964d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -225,6 +225,10 @@ if(DTO_BUILD_TESTS) endif() endif() +include(GNUInstallDirs) + +install(FILES dto.h DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}) + # Install and export the library install(TARGETS dto EXPORT DTOTargets diff --git a/Makefile b/Makefile index 06cc2e1..d0223b5 100644 --- a/Makefile +++ b/Makefile @@ -14,6 +14,7 @@ libdto_nostats: dto.c install: cp libdto.so.1.0 /usr/lib64/ + cp dto.h /usr/include/ ln -sf /usr/lib64/libdto.so.1.0 /usr/lib64/libdto.so.1 ln -sf /usr/lib64/libdto.so.1.0 /usr/lib64/libdto.so diff --git a/dto.c b/dto.c index 5eb211e..3832818 100644 --- a/dto.c +++ b/dto.c @@ -16,6 +16,8 @@ #include #include #include + +#include "dto.h" #include #include #include @@ -82,6 +84,7 @@ struct dto_wq { uint64_t dsa_gencap; int wq_size; uint32_t max_transfer_size; + uint32_t max_batch_size; int wq_fd; void *wq_portal; bool wq_mmapped; @@ -998,6 +1001,12 @@ static int dsa_init_from_wq_list(char *wq_list) goto fail_wq; } + wqs[num_wqs].max_batch_size = dto_get_param_ullong(dir_fd, "max_batch_size", &rc); + if (rc) { + close(dir_fd); + goto fail_wq; + } + dto_get_param_string(dir_fd, "mode", wq_mode); if (wq_mode[0] == '\0') { @@ -1161,6 +1170,7 @@ static int dsa_init_from_accfg(void) wqs[num_wqs].wq_size = accfg_wq_get_size(wq); wqs[num_wqs].max_transfer_size = accfg_wq_get_max_transfer_size(wq); + wqs[num_wqs].max_batch_size = accfg_wq_get_max_batch_size(wq); wqs[num_wqs].acc_wq = wq; wqs[num_wqs].dsa_gencap = accfg_device_get_gen_cap(device); @@ -1630,15 +1640,18 @@ static __always_inline struct dto_wq *get_wq(void* buf) return wq; } +/* 8-byte MEMFILL pattern for the byte value c (like memset) */ +static __always_inline uint64_t memfill_pattern(int c) +{ + return 0x0101010101010101ULL * (uint8_t)c; +} + static void dto_memset(void *s, int c, size_t n, int *result) { - uint64_t memset_pattern; + uint64_t memset_pattern = memfill_pattern(c); size_t cpu_size, dsa_size; struct dto_wq *wq = get_wq(s); - for (int i = 0; i < 8; ++i) - ((uint8_t *) &memset_pattern)[i] = (uint8_t) c; - thr_desc.opcode = DSA_OPCODE_MEMFILL; thr_desc.flags = IDXD_OP_FLAG_CRAV | IDXD_OP_FLAG_RCR; if (dto_dsa_cc && (wq->dsa_gencap & GENCAP_CC_MEMORY)) @@ -2150,3 +2163,260 @@ int memcmp(const void *s1, const void *s2, size_t n) } return ret; } + +/******************************************************************************* + * Explicit asynchronous submit/poll API (see dto.h). + ******************************************************************************/ + +/* With the default CRC flags the device inverts and bit-reflects both the + * seed and the result, so a seed of 0 yields the standard CRC32C + * (Castagnoli, init 0xFFFFFFFF, final inversion) and crc_val can be + * returned as is. */ +#define DSA_CRC32C_SEED 0u + +/* ENQCMD to a shared WQ can transiently fail when the queue is full; + * retry briefly before giving up so momentary bursts don't push work + * back onto the CPU. */ +#define ASYNC_SUBMIT_RETRIES 16 + +/* Failed async completions are logged at most this many times per process: + * poll may legitimately be called repeatedly on a failed op. */ +#define ASYNC_FAIL_LOG_LIMIT 3 + +struct dto_async_op_impl { + struct dsa_hw_desc desc; /* 64 bytes, 64-aligned via dto_async_op */ + struct dsa_completion_record comp __attribute__((aligned(32))); +}; +_Static_assert(sizeof(struct dto_async_op_impl) <= sizeof(dto_async_op), + "dto_async_op opaque storage too small"); +_Static_assert(sizeof(struct dsa_hw_desc) == 64, "unexpected descriptor size"); + +static __always_inline uint32_t async_desc_flags(const struct dto_wq *wq, + unsigned int flags, + bool has_dest) +{ + uint32_t f = IDXD_OP_FLAG_CRAV | IDXD_OP_FLAG_RCR; + + if (flags & DTO_SUBMIT_BOF) + f |= IDXD_OP_FLAG_BOF; + /* CC is only legal for operations with a destination; CRC Generation + * would be failed by the device with DSA_COMP_INVALID_FLAGS. */ + if ((flags & DTO_SUBMIT_CC) && has_dest && + (wq->dsa_gencap & GENCAP_CC_MEMORY)) + f |= IDXD_OP_FLAG_CC; + return f; +} + +static int async_submit(struct dto_wq *wq, struct dsa_hw_desc *desc) +{ + for (int attempt = 0; attempt < ASYNC_SUBMIT_RETRIES; attempt++) { + int rc = dsa_submit(wq, desc); + + if (rc == SUCCESS) + return DTO_ASYNC_SUBMITTED; + if (rc != RETRY) + break; + _mm_pause(); + } + return DTO_ASYNC_FALLBACK; +} + +/* Completion record status -> DTO_ASYNC_{PENDING,DONE,FAILED} */ +static int async_comp_result(const struct dsa_completion_record *comp, + uint32_t opcode, uint32_t size) +{ + static int fail_logged; + uint8_t status = __atomic_load_n((const uint8_t *)&comp->status, + __ATOMIC_ACQUIRE); + + if (status == 0) + return DTO_ASYNC_PENDING; + if (likely((status & DSA_COMP_STATUS_MASK) == DSA_COMP_SUCCESS)) + return DTO_ASYNC_DONE; + if (fail_logged < ASYNC_FAIL_LOG_LIMIT && + __atomic_fetch_add(&fail_logged, 1, __ATOMIC_RELAXED) < + ASYNC_FAIL_LOG_LIMIT) + LOG_ERROR("async op failed with status %x (opcode %x, size %x)\n", + status, opcode, size); + return DTO_ASYNC_FAILED; +} + +/* src_or_pattern is the source address, or the fill pattern for MEMFILL + * (both occupy the same descriptor slot). */ +static int dto_submit_async_common(dto_async_op *op, uint32_t opcode, + void *dest, uint64_t src_or_pattern, + size_t n, unsigned int flags) +{ + struct dto_async_op_impl *impl = (struct dto_async_op_impl *)op; + struct dto_wq *wq; + + if (!dto_initialized || use_std_lib_calls) + return DTO_ASYNC_FALLBACK; + /* The completion record must be 32-byte aligned. dto_async_op is + * declared 64-byte aligned, but a malloc'ed op need not be. */ + if (n == 0 || ((uintptr_t)op & 31) != 0) + return DTO_ASYNC_FALLBACK; + wq = get_wq(dest ? dest : (void *)(uintptr_t)src_or_pattern); + if (unlikely(wq == NULL) || n > wq->max_transfer_size) + return DTO_ASYNC_FALLBACK; + + orig_memset(&impl->desc, 0, sizeof(impl->desc)); + impl->desc.opcode = opcode; + impl->desc.flags = async_desc_flags(wq, flags, dest != NULL); + impl->desc.completion_addr = (uint64_t)&impl->comp; + impl->desc.xfer_size = (uint32_t)n; + impl->desc.src_addr = src_or_pattern; + impl->desc.dst_addr = (uint64_t)dest; + if (opcode == DSA_OPCODE_COPY_CRC || opcode == DSA_OPCODE_CRCGEN) + impl->desc.crc_seed = DSA_CRC32C_SEED; + impl->comp.status = 0; + + return async_submit(wq, &impl->desc); +} + +__attribute__((visibility("default"))) +int dto_submit_memcpy(dto_async_op *op, void *dest, const void *src, + size_t n, unsigned int flags) +{ + return dto_submit_async_common(op, DSA_OPCODE_MEMMOVE, dest, + (uint64_t)src, n, flags); +} + +__attribute__((visibility("default"))) +int dto_submit_memset(dto_async_op *op, void *dest, int c, size_t n, + unsigned int flags) +{ + return dto_submit_async_common(op, DSA_OPCODE_MEMFILL, dest, + memfill_pattern(c), n, flags); +} + +__attribute__((visibility("default"))) +int dto_submit_memcpy_crc(dto_async_op *op, void *dest, const void *src, + size_t n, unsigned int flags) +{ + return dto_submit_async_common(op, DSA_OPCODE_COPY_CRC, dest, + (uint64_t)src, n, flags); +} + +__attribute__((visibility("default"))) +int dto_submit_crc(dto_async_op *op, const void *src, size_t n, + unsigned int flags) +{ + return dto_submit_async_common(op, DSA_OPCODE_CRCGEN, NULL, + (uint64_t)src, n, flags); +} + +__attribute__((visibility("default"))) +int dto_async_poll(dto_async_op *op) +{ + struct dto_async_op_impl *impl = (struct dto_async_op_impl *)op; + + return async_comp_result(&impl->comp, impl->desc.opcode, + impl->desc.xfer_size); +} + +__attribute__((visibility("default"))) +uint32_t dto_async_crc_val(const dto_async_op *op) +{ + const struct dto_async_op_impl *impl = + (const struct dto_async_op_impl *)op; + return (uint32_t)impl->comp.crc_val; +} + +/* ---- Asynchronous batch copy (see dto.h) ---- */ + +struct dto_batch_op { + struct dsa_hw_desc desc __attribute__((aligned(64))); + struct dsa_completion_record comp __attribute__((aligned(32))); + struct dsa_hw_desc descs[DTO_BATCH_MAX] __attribute__((aligned(64))); + struct dsa_completion_record comps[DTO_BATCH_MAX] __attribute__((aligned(32))); +}; + +__attribute__((visibility("default"))) +dto_batch_op *dto_batch_op_new(void) +{ + void *p = NULL; + + if (posix_memalign(&p, 64, sizeof(struct dto_batch_op))) + return NULL; + /* usable before the constructor has run (orig_memset is not) */ + dto_internal_memset(p, 0, sizeof(struct dto_batch_op)); + return (dto_batch_op *)p; +} + +__attribute__((visibility("default"))) +void dto_batch_op_free(dto_batch_op *op) +{ + free(op); +} + +__attribute__((visibility("default"))) +int dto_submit_batch_copy(dto_batch_op *op, void **dst, void **src, + size_t *sizes, int count, unsigned int flags) +{ + struct dto_wq *wq; + uint32_t desc_flags; + + /* a DSA batch needs at least two descriptors */ + if (unlikely(!dto_initialized || use_std_lib_calls || + count < 2 || count > DTO_BATCH_MAX)) + return DTO_ASYNC_FALLBACK; + wq = get_wq(dst[0]); + if (unlikely(wq == NULL) || (uint32_t)count > wq->max_batch_size) + return DTO_ASYNC_FALLBACK; + /* validate everything before touching the op so FALLBACK leaves it + * untouched */ + for (int i = 0; i < count; i++) { + if (sizes[i] == 0 || sizes[i] > wq->max_transfer_size) + return DTO_ASYNC_FALLBACK; + } + + desc_flags = async_desc_flags(wq, flags, true); + orig_memset(op->descs, 0, sizeof(op->descs[0]) * count); + for (int i = 0; i < count; i++) { + struct dsa_hw_desc *desc = &op->descs[i]; + + desc->opcode = DSA_OPCODE_MEMMOVE; + desc->flags = desc_flags; + desc->src_addr = (uint64_t)src[i]; + desc->dst_addr = (uint64_t)dst[i]; + desc->xfer_size = (uint32_t)sizes[i]; + desc->completion_addr = (uint64_t)&op->comps[i]; + op->comps[i].status = 0; + } + orig_memset(&op->desc, 0, sizeof(op->desc)); + op->desc.opcode = DSA_OPCODE_BATCH; + op->desc.flags = IDXD_OP_FLAG_CRAV | IDXD_OP_FLAG_RCR; + op->desc.desc_list_addr = (uint64_t)op->descs; + op->desc.desc_count = count; + op->desc.completion_addr = (uint64_t)&op->comp; + op->comp.status = 0; + + return async_submit(wq, &op->desc); +} + +__attribute__((visibility("default"))) +int dto_batch_poll(dto_batch_op *op) +{ + int rc = async_comp_result(&op->comp, DSA_OPCODE_BATCH, + op->desc.desc_count); + + if (rc != DTO_ASYNC_FAILED) + return rc; + + /* redo the copies the device did not complete on the CPU */ + for (uint32_t i = 0; i < op->desc.desc_count; i++) { + const struct dsa_hw_desc *desc = &op->descs[i]; + + if ((op->comps[i].status & DSA_COMP_STATUS_MASK) != + DSA_COMP_SUCCESS) + orig_memcpy((void *)(uintptr_t)desc->dst_addr, + (const void *)(uintptr_t)desc->src_addr, + desc->xfer_size); + } + /* Mark the batch complete so a later poll (possibly from another + * thread) doesn't redo the copies from a source the caller may + * already have reused. */ + __atomic_store_n(&op->comp.status, DSA_COMP_SUCCESS, __ATOMIC_RELEASE); + return DTO_ASYNC_DONE; +} diff --git a/dto.h b/dto.h new file mode 100644 index 0000000..3f5176e --- /dev/null +++ b/dto.h @@ -0,0 +1,139 @@ +/******************************************************************************* + * Copyright (C) 2026 Intel Corporation + * + * SPDX-License-Identifier: MIT + * + * dto.h - Public asynchronous offload API for libdto. + * + * In addition to transparently accelerating memcpy/memmove/memset/memcmp via + * LD_PRELOAD, libdto exposes an explicit submit/poll API for applications + * that want to overlap accelerator data movement with CPU work. + * + * Lifecycle: + * + * dto_async_op op; + * if (dto_submit_memcpy(&op, dst, src, n, 1) == DTO_ASYNC_SUBMITTED) { + * ... other CPU work ... + * while (dto_async_poll(&op) == DTO_ASYNC_PENDING) + * ; // pause/yield + * if (dto_async_poll(&op) != DTO_ASYNC_DONE) + * memcpy(dst, src, n); // redo on the CPU + * } else { // DTO_ASYNC_FALLBACK: nothing submitted + * memcpy(dst, src, n); + * } + * + * The operation state is caller-allocated, so it can outlive the submitting + * call and be polled from a different thread; nothing is stored in + * thread-local state. dto_async_op is declared 64-byte aligned, which + * automatic and static storage honour; heap-allocated ops must come from + * aligned_alloc/posix_memalign (plain malloc does not guarantee it). An op + * that is not at least 32-byte aligned is rejected with DTO_ASYNC_FALLBACK. + * + * Submission returns DTO_ASYNC_FALLBACK when the accelerator is unavailable + * (not initialized, disabled via DTO_USESTDC_CALLS, no usable work queue) or + * the request is out of range (n == 0, or larger than the work queue's + * maximum transfer size). Nothing is copied in that case: the caller performs + * the operation on the CPU. Unlike the transparent path, explicit submits are + * not subject to the size heuristics (DTO_MIN_BYTES / auto-tuning): callers + * decide what to offload. + * + * On DTO_ASYNC_FAILED (e.g. a page fault the device could not resolve) the + * destination contents are unspecified; redo the whole operation on the CPU. + * + * @cache_control (DTO_SUBMIT_CC): directs the operation's output toward the + * CPU cache (IDXD_OP_FLAG_CC) when the device supports it, for destinations + * that will be read again soon. + * + * DTO_SUBMIT_BOF makes the device block on page faults and wait for the + * kernel to resolve them, instead of aborting the operation (which surfaces + * as DTO_ASYNC_FAILED from dto_async_poll). Use it when the buffers may not + * be fully faulted in. The work queue must be configured with + * block-on-fault enabled ("accel-config config-wq --block-on-fault=1"), + * otherwise the device rejects the descriptor and the operation fails. + ******************************************************************************/ + +#ifndef DTO_H +#define DTO_H + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/* Caller-allocated operation state (opaque). */ +typedef struct dto_async_op { + unsigned char opaque[192] __attribute__((aligned(64))); +} dto_async_op; + +/* Submission flags */ +#define DTO_SUBMIT_CC (1u << 0) /* see @cache_control above */ +#define DTO_SUBMIT_BOF (1u << 1) /* block on page faults (see below) */ + +/* Submit results */ +#define DTO_ASYNC_SUBMITTED 0 +#define DTO_ASYNC_FALLBACK (-1) + +/* Poll results */ +#define DTO_ASYNC_PENDING 0 +#define DTO_ASYNC_DONE 1 +#define DTO_ASYNC_FAILED (-1) + +/* Copy n bytes from src to dest. */ +int dto_submit_memcpy(dto_async_op *op, void *dest, const void *src, + size_t n, unsigned int flags); + +/* Fill n bytes of dest with the byte value c (like memset). */ +int dto_submit_memset(dto_async_op *op, void *dest, int c, size_t n, + unsigned int flags); + +/* Copy n bytes from src to dest and compute the CRC32C of the data. + * The CRC value (read with dto_async_crc_val after DTO_ASYNC_DONE) is the + * standard CRC32C (Castagnoli, seed 0xFFFFFFFF, final inversion), i.e. it + * matches common software crc32c implementations. */ +int dto_submit_memcpy_crc(dto_async_op *op, void *dest, const void *src, + size_t n, unsigned int flags); + +/* Compute the CRC32C of n bytes at src without copying. */ +int dto_submit_crc(dto_async_op *op, const void *src, size_t n, + unsigned int flags); + +/* Poll a submitted operation. Returns DTO_ASYNC_PENDING, DTO_ASYNC_DONE or + * DTO_ASYNC_FAILED. May be called repeatedly, from any thread; once an op + * has left PENDING its result is stable. */ +int dto_async_poll(dto_async_op *op); + +/* CRC32C result of a completed dto_submit_memcpy_crc/dto_submit_crc. + * Valid only after dto_async_poll returned DTO_ASYNC_DONE. */ +uint32_t dto_async_crc_val(const dto_async_op *op); + +/* ---- Asynchronous batch copy ---- + * + * Submits up to DTO_BATCH_MAX copies as one DSA batch descriptor. The caller + * owns the op (dto_batch_op_new / dto_batch_op_free) and may reuse it for + * consecutive batches. A DSA batch requires at least two descriptors, so + * count < 2 returns DTO_ASYNC_FALLBACK, as does count > DTO_BATCH_MAX or + * count > the work queue's configured max_batch_size. + * + * dto_submit_batch_copy returns DTO_ASYNC_SUBMITTED or DTO_ASYNC_FALLBACK + * (nothing submitted, nothing copied). dto_batch_poll returns + * DTO_ASYNC_PENDING or DTO_ASYNC_DONE; copies the accelerator failed are + * redone on the CPU before DONE is returned, so DONE means every copy is + * complete. Once DONE, further polls return DONE without touching the + * buffers again. */ +#define DTO_BATCH_MAX 64 + +typedef struct dto_batch_op dto_batch_op; + +dto_batch_op *dto_batch_op_new(void); +void dto_batch_op_free(dto_batch_op *op); +int dto_submit_batch_copy(dto_batch_op *op, void **dst, void **src, + size_t *sizes, int count, unsigned int flags); +int dto_batch_poll(dto_batch_op *op); + +#ifdef __cplusplus +} +#endif + +#endif /* DTO_H */ diff --git a/tests/test_functional.c b/tests/test_functional.c index f07affe..eb667fd 100644 --- a/tests/test_functional.c +++ b/tests/test_functional.c @@ -19,6 +19,8 @@ ******************************************************************************/ #include "dto_test_utils.h" +#include "../dto.h" +#include #include #include @@ -507,6 +509,229 @@ static int test_multithread(void) /* ---- Test runner ---- */ +/* ---- Explicit asynchronous API (dto.h) ---- */ + +/* Standard CRC32C (Castagnoli): init 0xFFFFFFFF, final inversion, the + * convention dto.h documents for dto_async_crc_val. */ +__attribute__((target("sse4.2"))) +static uint32_t ref_crc32c(const void *buf, size_t n) +{ + const uint8_t *p = buf; + uint32_t crc = 0xFFFFFFFFu; + for (size_t i = 0; i < n; i++) + crc = _mm_crc32_u8(crc, p[i]); + return ~crc; +} + +/* Poll until poll_expr leaves PENDING and yield its result. A submitted op + * owns its (stack-allocated) state until it completes, so a stall past the + * 10s safety timeout aborts the process rather than returning into a test + * that would let the device write into a dead stack frame. */ +#define ASYNC_WAIT(poll_expr) ({ \ + struct timespec _start, _now; \ + int _rc; \ + clock_gettime(CLOCK_MONOTONIC, &_start); \ + while ((_rc = (poll_expr)) == DTO_ASYNC_PENDING) { \ + clock_gettime(CLOCK_MONOTONIC, &_now); \ + if (_now.tv_sec - _start.tv_sec > 10) { \ + fprintf(stderr, " FATAL: async op still pending after 10s at %s:%d\n", \ + __FILE__, __LINE__); \ + abort(); \ + } \ + } \ + _rc; }) + +/* Submit a memcpy with the given flags and verify the data afterwards, + * whichever of the three documented outcomes (DONE, FAILED + CPU redo, + * FALLBACK + CPU copy) occurs. */ +static int check_async_memcpy(unsigned int flags) +{ + const size_t n = 1 << 20; + uint8_t *src = malloc(n), *dst = malloc(n); + dto_async_op op; + int rc; + + ASSERT_TRUE(src && dst); + fill_pattern(src, n); + clear_buf(dst, n); + + rc = dto_submit_memcpy(&op, dst, src, n, flags); + if (rc == DTO_ASYNC_SUBMITTED) { + rc = ASYNC_WAIT(dto_async_poll(&op)); + ASSERT_TRUE(rc == DTO_ASYNC_DONE || rc == DTO_ASYNC_FAILED); + if (rc != DTO_ASYNC_DONE) + memcpy(dst, src, n); /* documented caller fallback */ + } else { + ASSERT_EQ(rc, DTO_ASYNC_FALLBACK); + memcpy(dst, src, n); + } + ASSERT_TRUE(verify_equal(dst, src, n)); + free(src); + free(dst); + return 0; +} + +static int test_async_memcpy(void) +{ + return check_async_memcpy(DTO_SUBMIT_CC | DTO_SUBMIT_BOF); +} + +static int test_async_memcpy_no_bof(void) +{ + /* Without DTO_SUBMIT_BOF a page fault aborts the operation and poll + * reports DTO_ASYNC_FAILED; the caller redoes the copy on the CPU. + * Either outcome is valid here, the data must be right afterwards. */ + return check_async_memcpy(0); +} + +static int test_async_memset(void) +{ + const size_t n = 1 << 20; + uint8_t *dst = malloc(n); + dto_async_op op; + int rc; + + ASSERT_TRUE(dst != NULL); + clear_buf(dst, n); + + rc = dto_submit_memset(&op, dst, 0x5A, n, DTO_SUBMIT_BOF); + if (rc == DTO_ASYNC_SUBMITTED) { + rc = ASYNC_WAIT(dto_async_poll(&op)); + ASSERT_TRUE(rc == DTO_ASYNC_DONE || rc == DTO_ASYNC_FAILED); + if (rc != DTO_ASYNC_DONE) + memset(dst, 0x5A, n); + } else { + ASSERT_EQ(rc, DTO_ASYNC_FALLBACK); + memset(dst, 0x5A, n); + } + ASSERT_TRUE(verify_set(dst, 0x5A, n)); + free(dst); + return 0; +} + +/* BOF is rejected by the device on a WQ configured without block-on-fault + * (dto.h), so like the other tests the CRC tests accept DTO_ASYNC_FAILED; + * the CRC value is only checked when the device completed the op. */ +static int test_async_crc(void) +{ + const size_t n = 256 * 1024; + uint8_t *src = malloc(n); + dto_async_op op; + uint32_t expect; + int rc; + + ASSERT_TRUE(src != NULL); + fill_pattern(src, n); + expect = ref_crc32c(src, n); + + rc = dto_submit_crc(&op, src, n, DTO_SUBMIT_BOF); + if (rc == DTO_ASYNC_SUBMITTED) { + rc = ASYNC_WAIT(dto_async_poll(&op)); + ASSERT_TRUE(rc == DTO_ASYNC_DONE || rc == DTO_ASYNC_FAILED); + if (rc == DTO_ASYNC_DONE) + ASSERT_EQ(dto_async_crc_val(&op), expect); + } else { + ASSERT_EQ(rc, DTO_ASYNC_FALLBACK); + } + free(src); + return 0; +} + +static int test_async_memcpy_crc(void) +{ + const size_t n = 256 * 1024; + uint8_t *src = malloc(n), *dst = malloc(n); + dto_async_op op; + uint32_t expect; + int rc; + + ASSERT_TRUE(src && dst); + fill_pattern(src, n); + clear_buf(dst, n); + expect = ref_crc32c(src, n); + + rc = dto_submit_memcpy_crc(&op, dst, src, n, DTO_SUBMIT_CC | DTO_SUBMIT_BOF); + if (rc == DTO_ASYNC_SUBMITTED) { + rc = ASYNC_WAIT(dto_async_poll(&op)); + ASSERT_TRUE(rc == DTO_ASYNC_DONE || rc == DTO_ASYNC_FAILED); + if (rc == DTO_ASYNC_DONE) + ASSERT_EQ(dto_async_crc_val(&op), expect); + else + memcpy(dst, src, n); + } else { + ASSERT_EQ(rc, DTO_ASYNC_FALLBACK); + memcpy(dst, src, n); + } + ASSERT_TRUE(verify_equal(dst, src, n)); + free(src); + free(dst); + return 0; +} + +static int test_async_bad_args(void) +{ + uint8_t a[64], b[64]; + void *dstv[1] = { a }, *srcv[1] = { b }; + size_t sizes[1] = { 64 }; + dto_async_op op; + dto_batch_op *bop; + + /* zero length never reaches the device */ + ASSERT_EQ(dto_submit_memcpy(&op, a, b, 0, 0), DTO_ASYNC_FALLBACK); + ASSERT_EQ(dto_submit_crc(&op, b, 0, 0), DTO_ASYNC_FALLBACK); + + /* a DSA batch needs at least two descriptors */ + bop = dto_batch_op_new(); + ASSERT_TRUE(bop != NULL); + ASSERT_EQ(dto_submit_batch_copy(bop, dstv, srcv, sizes, 1, 0), + DTO_ASYNC_FALLBACK); + dto_batch_op_free(bop); + return 0; +} + +static int test_async_batch_copy(void) +{ + enum { COUNT = 8 }; + const size_t base = 16 * 1024; + uint8_t *src[COUNT], *dst[COUNT]; + void *srcv[COUNT], *dstv[COUNT]; + size_t sizes[COUNT]; + dto_batch_op *bop; + int rc; + + for (int i = 0; i < COUNT; i++) { + sizes[i] = base * (i + 1) + i * 64; /* varied, some unaligned */ + src[i] = malloc(sizes[i]); + dst[i] = malloc(sizes[i]); + ASSERT_TRUE(src[i] && dst[i]); + fill_pattern(src[i], sizes[i]); + clear_buf(dst[i], sizes[i]); + srcv[i] = src[i]; + dstv[i] = dst[i]; + } + + bop = dto_batch_op_new(); + ASSERT_TRUE(bop != NULL); + rc = dto_submit_batch_copy(bop, dstv, srcv, sizes, COUNT, DTO_SUBMIT_BOF); + if (rc == DTO_ASYNC_SUBMITTED) { + rc = ASYNC_WAIT(dto_batch_poll(bop)); + ASSERT_EQ(rc, DTO_ASYNC_DONE); + /* a completed batch stays DONE */ + ASSERT_EQ(dto_batch_poll(bop), DTO_ASYNC_DONE); + } else { + ASSERT_EQ(rc, DTO_ASYNC_FALLBACK); + for (int i = 0; i < COUNT; i++) + memcpy(dst[i], src[i], sizes[i]); + } + for (int i = 0; i < COUNT; i++) { + ASSERT_TRUE(verify_equal(dst[i], src[i], sizes[i])); + free(src[i]); + free(dst[i]); + } + dto_batch_op_free(bop); + return 0; +} + int main(void) { printf("DTO Functional Tests\n"); @@ -523,6 +748,13 @@ int main(void) TEST_ENTRY(test_zero_length), TEST_ENTRY(test_memcpy_unaligned), TEST_ENTRY(test_multithread), + TEST_ENTRY(test_async_memcpy), + TEST_ENTRY(test_async_memset), + TEST_ENTRY(test_async_crc), + TEST_ENTRY(test_async_memcpy_crc), + TEST_ENTRY(test_async_memcpy_no_bof), + TEST_ENTRY(test_async_bad_args), + TEST_ENTRY(test_async_batch_copy), {NULL, NULL} }; From df0111ad189118ef545bc77ca7076c9fb812c5a0 Mon Sep 17 00:00:00 2001 From: Daniel Byrne Date: Wed, 21 Oct 2026 08:20:05 -0700 Subject: [PATCH 2/5] Unify the explicit API with the cachelib-navy and ch-async-ops branches Bring the asynchronous API work from the two development branches into one interface, keeping this PR's conventions: every submit takes a flags argument (DTO_SUBMIT_CC, DTO_SUBMIT_BOF), DTO_ASYNC_FALLBACK means nothing was done, and explicit submits are not gated by DTO_MIN_BYTES. From ch-async-ops (Cloud Hypervisor): - dto_submit_memfill, dto_submit_compare, dto_submit_compval and dto_submit_transl_fetch, with dto_async_result, dto_async_bytes_completed and dto_async_status. - The dto_batch builder: dto_batch_create (up to DTO_BATCH_MAX = 1024 operations), add_memmove/memfill/dualcast/crc/memcpy_crc/compare/ compval, submit, poll and per-operation status, result, bytes_completed and crc. A single operation goes out as a plain descriptor. It replaces dto_batch_op; dto_submit_batch_copy remains as a convenience on top of it. From cachelib-navy: - Seeded CRC32C in the iSCSI presentation, dto_submit_memcpy_crc32c, dto_submit_crc32c and dto_async_crc32c_val, and the synchronous dto_crc32c_with_seed (gated by the new DTO_CRC_MIN_BYTES, default DTO_MIN_BYTES). - dto_async_wait and dto_batch_wait, which wait with the configured DTO_WAIT_METHOD and are no-ops for ops that were never submitted, and dto_wait_blocks (0 until a blocking wait method exists). dto_async_crc_val and dto_batch_crc now return the raw CRC32C (the SSE4.2 crc32 loop from 0, no final inversion), which both branches and their users' CPU fallbacks use, instead of standard CRC32C; the seeded calls provide the standard form. dto_batch_poll redoes every operation the device did not complete on the CPU before returning DONE, results and CRCs included, so DONE means every result is valid; dto_batch_status still reports the device's status. A move the device stopped partway is resumed from where it stopped, in the direction it was copying, because with overlapping buffers the device has already overwritten part of the source. One poller repairs; concurrent pollers see PENDING until it is done. DTO_SUBMIT_BOF is passed to the device as given. dto.h now spells out what that means for each work queue configuration: with block-on-fault enabled the device waits for the fault; without it a BOF submit fails with DSA_COMP_INVALID_FLAGS; and without the flag a page fault is a partial completion, which for an overlapping move has to be finished from where the device stopped rather than redone. Build with DTO_NO_LIBC_INTERPOSE to leave out the memcpy, memset, memmove and memcmp interposers (libdto_explicit). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Mv7xKtQPkGoXQUvvuuhAEz --- dto.c | 827 +++++++++++++++++++++++++++++++++++++++++++++++++++------- dto.h | 222 +++++++++++++--- 2 files changed, 905 insertions(+), 144 deletions(-) diff --git a/dto.c b/dto.c index 3832818..24b606b 100644 --- a/dto.c +++ b/dto.c @@ -66,7 +66,9 @@ #define NSEC_PER_MSEC (NSEC_PER_SEC/MSEC_PER_SEC) // thread specific variables +#ifndef DTO_NO_LIBC_INTERPOSE static __thread struct dsa_hw_desc thr_desc; +#endif static __thread struct dsa_completion_record thr_comp __attribute__((aligned(32))); static __thread uint64_t thr_bytes_completed; @@ -136,6 +138,9 @@ static atomic_uchar dto_initializing; static uint8_t use_std_lib_calls; static enum numa_aware is_numa_aware; static size_t dsa_min_size = DTO_DEFAULT_MIN_SIZE; +/* DTO_CRC_MIN_BYTES: smallest dto_crc32c_with_seed() sent to DSA; 0 follows + * dsa_min_size */ +static size_t crc_min_size; static int wait_method = WAIT_BUSYPOLL; static size_t cpu_size_fraction; // range of values is 0 to 99 @@ -1494,6 +1499,15 @@ static int init_dto(void) dsa_min_size = DTO_DEFAULT_MIN_SIZE; } + env_str = getenv("DTO_CRC_MIN_BYTES"); + + if (env_str != NULL) { + errno = 0; + crc_min_size = strtoul(env_str, NULL, 10); + if (errno) + crc_min_size = 0; + } + double cpu_size_fraction_float = 0.0; env_str = getenv("DTO_CPU_SIZE_FRACTION"); @@ -1646,6 +1660,11 @@ static __always_inline uint64_t memfill_pattern(int c) return 0x0101010101010101ULL * (uint8_t)c; } +/* libdto_explicit (DTO_NO_LIBC_INTERPOSE) leaves libc's memset, memcpy, + * memmove and memcmp alone: programs that use it call the dto_* API below + * directly. */ +#ifndef DTO_NO_LIBC_INTERPOSE + static void dto_memset(void *s, int c, size_t n, int *result) { uint64_t memset_pattern = memfill_pattern(c); @@ -2164,39 +2183,74 @@ int memcmp(const void *s1, const void *s2, size_t n) return ret; } +#endif /* DTO_NO_LIBC_INTERPOSE */ + /******************************************************************************* * Explicit asynchronous submit/poll API (see dto.h). ******************************************************************************/ -/* With the default CRC flags the device inverts and bit-reflects both the - * seed and the result, so a seed of 0 yields the standard CRC32C - * (Castagnoli, init 0xFFFFFFFF, final inversion) and crc_val can be - * returned as is. */ -#define DSA_CRC32C_SEED 0u +/* CRC presentations. With its default CRC flags the device inverts both the + * seed and the result: the iSCSI presentation, in which seed 0 gives the + * standard CRC32C and a previous result chains. The crc32c calls return it + * unchanged. The raw CRC32C of dto_async_crc_val/dto_batch_crc -- the SSE4.2 + * crc32 loop from 0, no final inversion -- is the same computation seeded + * with ~0 and inverted back. */ +#define DSA_CRC_SEED_RAW 0xFFFFFFFFu +#define DSA_CRC_VAL_TO_RAW(v) ((uint32_t)~(uint32_t)(v)) /* ENQCMD to a shared WQ can transiently fail when the queue is full; * retry briefly before giving up so momentary bursts don't push work - * back onto the CPU. */ + * back onto the CPU. A batch carries more work, so it tries longer. */ #define ASYNC_SUBMIT_RETRIES 16 +#define BATCH_SUBMIT_RETRIES 64 /* Failed async completions are logged at most this many times per process: * poll may legitimately be called repeatedly on a failed op. */ #define ASYNC_FAIL_LOG_LIMIT 3 +/* dto_async_wait() must not wait for a status byte no device will ever + * write: an op whose submit fell back, or one never submitted at all (ops + * are usually stack variables, so their storage starts out as garbage). + * Every submit clears the mark first and sets it once the descriptor is + * enqueued; garbage matching it by chance is a 2^-32 event. */ +#define DTO_OP_SUBMITTED_MAGIC 0x64746F53u /* "dtoS" */ + struct dto_async_op_impl { struct dsa_hw_desc desc; /* 64 bytes, 64-aligned via dto_async_op */ struct dsa_completion_record comp __attribute__((aligned(32))); + uint32_t submitted; }; _Static_assert(sizeof(struct dto_async_op_impl) <= sizeof(dto_async_op), "dto_async_op opaque storage too small"); _Static_assert(sizeof(struct dsa_hw_desc) == 64, "unexpected descriptor size"); +typedef uint64_t __attribute__((may_alias, aligned(1))) unaligned_u64; + +/* CRC32C on the CPU in the device's presentation for the given seed, so a + * value the library computes matches what the device returns for the same + * descriptor. */ +__attribute__((target("sse4.2"))) +static uint32_t cpu_crc32c(uint32_t seed, const void *buf, size_t n) +{ + const uint8_t *p = buf; + uint64_t crc = (uint32_t)~seed; + + for (; n >= 8; n -= 8, p += 8) + crc = _mm_crc32_u64(crc, *(const unaligned_u64 *)p); + for (; n; n--) + crc = _mm_crc32_u8((uint32_t)crc, *p++); + return ~(uint32_t)crc; +} + static __always_inline uint32_t async_desc_flags(const struct dto_wq *wq, unsigned int flags, bool has_dest) { uint32_t f = IDXD_OP_FLAG_CRAV | IDXD_OP_FLAG_RCR; + /* Passed as given: a work queue configured without block-on-fault + * fails the descriptor with DSA_COMP_INVALID_FLAGS, and without the + * flag a page fault stops the operation partway (see dto.h). */ if (flags & DTO_SUBMIT_BOF) f |= IDXD_OP_FLAG_BOF; /* CC is only legal for operations with a destination; CRC Generation @@ -2207,9 +2261,10 @@ static __always_inline uint32_t async_desc_flags(const struct dto_wq *wq, return f; } -static int async_submit(struct dto_wq *wq, struct dsa_hw_desc *desc) +static int async_submit(struct dto_wq *wq, struct dsa_hw_desc *desc, + int retries) { - for (int attempt = 0; attempt < ASYNC_SUBMIT_RETRIES; attempt++) { + for (int attempt = 0; attempt < retries; attempt++) { int rc = dsa_submit(wq, desc); if (rc == SUCCESS) @@ -2221,11 +2276,21 @@ static int async_submit(struct dto_wq *wq, struct dsa_hw_desc *desc) return DTO_ASYNC_FALLBACK; } +static void async_log_failure(uint8_t status, uint32_t opcode, uint32_t size) +{ + static int fail_logged; + + if (fail_logged < ASYNC_FAIL_LOG_LIMIT && + __atomic_fetch_add(&fail_logged, 1, __ATOMIC_RELAXED) < + ASYNC_FAIL_LOG_LIMIT) + LOG_ERROR("async op failed with status %x (opcode %x, size %x)\n", + status, opcode, size); +} + /* Completion record status -> DTO_ASYNC_{PENDING,DONE,FAILED} */ static int async_comp_result(const struct dsa_completion_record *comp, uint32_t opcode, uint32_t size) { - static int fail_logged; uint8_t status = __atomic_load_n((const uint8_t *)&comp->status, __ATOMIC_ACQUIRE); @@ -2233,77 +2298,179 @@ static int async_comp_result(const struct dsa_completion_record *comp, return DTO_ASYNC_PENDING; if (likely((status & DSA_COMP_STATUS_MASK) == DSA_COMP_SUCCESS)) return DTO_ASYNC_DONE; - if (fail_logged < ASYNC_FAIL_LOG_LIMIT && - __atomic_fetch_add(&fail_logged, 1, __ATOMIC_RELAXED) < - ASYNC_FAIL_LOG_LIMIT) - LOG_ERROR("async op failed with status %x (opcode %x, size %x)\n", - status, opcode, size); + async_log_failure(status, opcode, size); return DTO_ASYNC_FAILED; } -/* src_or_pattern is the source address, or the fill pattern for MEMFILL - * (both occupy the same descriptor slot). */ -static int dto_submit_async_common(dto_async_op *op, uint32_t opcode, - void *dest, uint64_t src_or_pattern, - size_t n, unsigned int flags) +/* Common part of every single-op submit. NULL means fall back, with nothing + * submitted; otherwise the descriptor is set up but for its operands. hint + * picks the work queue (NUMA-aware selection uses its node). */ +static struct dto_wq *async_prepare(struct dto_async_op_impl *impl, + uint32_t opcode, const void *hint, + size_t n, unsigned int flags, bool has_dest) { - struct dto_async_op_impl *impl = (struct dto_async_op_impl *)op; struct dto_wq *wq; + impl->submitted = 0; if (!dto_initialized || use_std_lib_calls) - return DTO_ASYNC_FALLBACK; + return NULL; /* The completion record must be 32-byte aligned. dto_async_op is * declared 64-byte aligned, but a malloc'ed op need not be. */ - if (n == 0 || ((uintptr_t)op & 31) != 0) - return DTO_ASYNC_FALLBACK; - wq = get_wq(dest ? dest : (void *)(uintptr_t)src_or_pattern); + if (n == 0 || ((uintptr_t)impl & 31) != 0) + return NULL; + wq = get_wq((void *)hint); if (unlikely(wq == NULL) || n > wq->max_transfer_size) - return DTO_ASYNC_FALLBACK; + return NULL; orig_memset(&impl->desc, 0, sizeof(impl->desc)); impl->desc.opcode = opcode; - impl->desc.flags = async_desc_flags(wq, flags, dest != NULL); + impl->desc.flags = async_desc_flags(wq, flags, has_dest); impl->desc.completion_addr = (uint64_t)&impl->comp; impl->desc.xfer_size = (uint32_t)n; - impl->desc.src_addr = src_or_pattern; - impl->desc.dst_addr = (uint64_t)dest; - if (opcode == DSA_OPCODE_COPY_CRC || opcode == DSA_OPCODE_CRCGEN) - impl->desc.crc_seed = DSA_CRC32C_SEED; impl->comp.status = 0; + return wq; +} - return async_submit(wq, &impl->desc); +static int async_enqueue(struct dto_async_op_impl *impl, struct dto_wq *wq) +{ + if (async_submit(wq, &impl->desc, ASYNC_SUBMIT_RETRIES) != + DTO_ASYNC_SUBMITTED) + return DTO_ASYNC_FALLBACK; + impl->submitted = DTO_OP_SUBMITTED_MAGIC; + return DTO_ASYNC_SUBMITTED; } __attribute__((visibility("default"))) int dto_submit_memcpy(dto_async_op *op, void *dest, const void *src, size_t n, unsigned int flags) { - return dto_submit_async_common(op, DSA_OPCODE_MEMMOVE, dest, - (uint64_t)src, n, flags); + struct dto_async_op_impl *impl = (struct dto_async_op_impl *)op; + struct dto_wq *wq = async_prepare(impl, DSA_OPCODE_MEMMOVE, dest, n, + flags, true); + + if (wq == NULL) + return DTO_ASYNC_FALLBACK; + impl->desc.src_addr = (uint64_t)src; + impl->desc.dst_addr = (uint64_t)dest; + return async_enqueue(impl, wq); +} + +__attribute__((visibility("default"))) +int dto_submit_memfill(dto_async_op *op, void *dest, uint64_t pattern, + size_t n, unsigned int flags) +{ + struct dto_async_op_impl *impl = (struct dto_async_op_impl *)op; + struct dto_wq *wq = async_prepare(impl, DSA_OPCODE_MEMFILL, dest, n, + flags, true); + + if (wq == NULL) + return DTO_ASYNC_FALLBACK; + impl->desc.pattern = pattern; + impl->desc.dst_addr = (uint64_t)dest; + return async_enqueue(impl, wq); } __attribute__((visibility("default"))) int dto_submit_memset(dto_async_op *op, void *dest, int c, size_t n, unsigned int flags) { - return dto_submit_async_common(op, DSA_OPCODE_MEMFILL, dest, - memfill_pattern(c), n, flags); + return dto_submit_memfill(op, dest, memfill_pattern(c), n, flags); +} + +static int submit_crc_common(dto_async_op *op, uint32_t opcode, void *dest, + const void *src, size_t n, uint32_t seed, + unsigned int flags) +{ + struct dto_async_op_impl *impl = (struct dto_async_op_impl *)op; + struct dto_wq *wq = async_prepare(impl, opcode, dest ? dest : src, n, + flags, dest != NULL); + + if (wq == NULL) + return DTO_ASYNC_FALLBACK; + impl->desc.src_addr = (uint64_t)src; + impl->desc.dst_addr = (uint64_t)dest; + impl->desc.crc_seed = seed; + return async_enqueue(impl, wq); } __attribute__((visibility("default"))) int dto_submit_memcpy_crc(dto_async_op *op, void *dest, const void *src, size_t n, unsigned int flags) { - return dto_submit_async_common(op, DSA_OPCODE_COPY_CRC, dest, - (uint64_t)src, n, flags); + return submit_crc_common(op, DSA_OPCODE_COPY_CRC, dest, src, n, + DSA_CRC_SEED_RAW, flags); } __attribute__((visibility("default"))) int dto_submit_crc(dto_async_op *op, const void *src, size_t n, unsigned int flags) { - return dto_submit_async_common(op, DSA_OPCODE_CRCGEN, NULL, - (uint64_t)src, n, flags); + return submit_crc_common(op, DSA_OPCODE_CRCGEN, NULL, src, n, + DSA_CRC_SEED_RAW, flags); +} + +__attribute__((visibility("default"))) +int dto_submit_memcpy_crc32c(dto_async_op *op, void *dest, const void *src, + size_t n, uint32_t seed, unsigned int flags) +{ + return submit_crc_common(op, DSA_OPCODE_COPY_CRC, dest, src, n, seed, + flags); +} + +__attribute__((visibility("default"))) +int dto_submit_crc32c(dto_async_op *op, const void *src, size_t n, + uint32_t seed, unsigned int flags) +{ + return submit_crc_common(op, DSA_OPCODE_CRCGEN, NULL, src, n, seed, + flags); +} + +__attribute__((visibility("default"))) +int dto_submit_compare(dto_async_op *op, const void *src1, const void *src2, + size_t n, unsigned int flags) +{ + struct dto_async_op_impl *impl = (struct dto_async_op_impl *)op; + struct dto_wq *wq = async_prepare(impl, DSA_OPCODE_COMPARE, src1, n, + flags, false); + + if (wq == NULL) + return DTO_ASYNC_FALLBACK; + impl->desc.src_addr = (uint64_t)src1; + impl->desc.src2_addr = (uint64_t)src2; + return async_enqueue(impl, wq); +} + +__attribute__((visibility("default"))) +int dto_submit_compval(dto_async_op *op, const void *src, uint64_t pattern, + size_t n, unsigned int flags) +{ + struct dto_async_op_impl *impl = (struct dto_async_op_impl *)op; + struct dto_wq *wq = async_prepare(impl, DSA_OPCODE_COMPVAL, src, n, + flags, false); + + if (wq == NULL) + return DTO_ASYNC_FALLBACK; + impl->desc.src_addr = (uint64_t)src; + /* Compare Pattern takes its pattern from the second operand slot; + * the first one is the source (it is Memory Fill's pattern slot). */ + impl->desc.comp_pattern = pattern; + return async_enqueue(impl, wq); +} + +__attribute__((visibility("default"))) +int dto_submit_transl_fetch(dto_async_op *op, const void *addr, size_t n, + unsigned int flags) +{ + struct dto_async_op_impl *impl = (struct dto_async_op_impl *)op; + struct dto_wq *wq = async_prepare(impl, DSA_OPCODE_TRANSL_FETCH, addr, + n, flags, false); + + if (wq == NULL) + return DTO_ASYNC_FALLBACK; + /* region_size shares the xfer_size slot async_prepare filled in */ + impl->desc.transl_fetch_addr = (uint64_t)addr; + impl->desc.region_stride = 0; /* one contiguous region */ + return async_enqueue(impl, wq); } __attribute__((visibility("default"))) @@ -2315,108 +2482,566 @@ int dto_async_poll(dto_async_op *op) impl->desc.xfer_size); } +__attribute__((visibility("default"))) +void dto_async_wait(dto_async_op *op) +{ + struct dto_async_op_impl *impl = (struct dto_async_op_impl *)op; + + if (impl->submitted != DTO_OP_SUBMITTED_MAGIC) + return; + dsa_wait_no_adjust(&impl->comp.status); + /* order the status byte ahead of the caller's reads of the record */ + atomic_thread_fence(memory_order_acquire); +} + +__attribute__((visibility("default"))) +int dto_wait_blocks(void) +{ + return 0; +} + __attribute__((visibility("default"))) uint32_t dto_async_crc_val(const dto_async_op *op) +{ + const struct dto_async_op_impl *impl = + (const struct dto_async_op_impl *)op; + return DSA_CRC_VAL_TO_RAW(impl->comp.crc_val); +} + +__attribute__((visibility("default"))) +uint32_t dto_async_crc32c_val(const dto_async_op *op) { const struct dto_async_op_impl *impl = (const struct dto_async_op_impl *)op; return (uint32_t)impl->comp.crc_val; } -/* ---- Asynchronous batch copy (see dto.h) ---- */ +__attribute__((visibility("default"))) +int dto_async_result(const dto_async_op *op) +{ + const struct dto_async_op_impl *impl = + (const struct dto_async_op_impl *)op; + return impl->comp.result; +} + +__attribute__((visibility("default"))) +uint32_t dto_async_bytes_completed(const dto_async_op *op) +{ + const struct dto_async_op_impl *impl = + (const struct dto_async_op_impl *)op; + return impl->comp.bytes_completed; +} -struct dto_batch_op { - struct dsa_hw_desc desc __attribute__((aligned(64))); +__attribute__((visibility("default"))) +int dto_async_status(const dto_async_op *op) +{ + const struct dto_async_op_impl *impl = + (const struct dto_async_op_impl *)op; + return __atomic_load_n((const uint8_t *)&impl->comp.status, + __ATOMIC_ACQUIRE) & DSA_COMP_STATUS_MASK; +} + +__attribute__((visibility("default"))) +uint32_t dto_crc32c_with_seed(uint32_t seed, const void *src, size_t n) +{ + size_t min_size = crc_min_size ? crc_min_size : dsa_min_size; + dto_async_op op; + + /* No DTO_SUBMIT_BOF, so this works on any work queue: a page fault + * stops the device and the CRC is computed on the CPU instead. */ + if (n >= min_size && + dto_submit_crc32c(&op, src, n, seed, 0) == DTO_ASYNC_SUBMITTED) { + dto_async_wait(&op); + if (dto_async_poll(&op) == DTO_ASYNC_DONE) + return dto_async_crc32c_val(&op); + } + return cpu_crc32c(seed, src, n); +} + +/* ---- Batches (see dto.h) ---- */ + +/* Zero storage the library owns. Once DTO is up that is libc's memset, so it + * costs no detour through the interposer and a tiny DTO_MIN_BYTES never sends + * it to the device; before that, memset() itself, which copes with an + * uninitialized DTO (and is libc's in libdto_explicit). */ +static void lib_zero(void *p, size_t n) +{ + if (orig_memset) + orig_memset(p, 0, n); + else + memset(p, 0, n); +} + +enum batch_state { + BATCH_BUILDING = 0, /* accepting operations */ + BATCH_SUBMITTED, /* in flight */ + BATCH_REPAIRING, /* one poller is redoing failed operations */ + BATCH_DONE, +}; + +struct dto_batch { + struct dsa_hw_desc desc __attribute__((aligned(64))); /* the Batch descriptor */ struct dsa_completion_record comp __attribute__((aligned(32))); - struct dsa_hw_desc descs[DTO_BATCH_MAX] __attribute__((aligned(64))); - struct dsa_completion_record comps[DTO_BATCH_MAX] __attribute__((aligned(32))); + struct dsa_hw_desc *descs; /* [cap], 64-byte aligned */ + struct dsa_completion_record *comps; /* [cap], 32-byte aligned */ + const void *hint; /* first operand: picks the WQ */ + int cap; + int count; + _Atomic int state; }; +/* The record that completes the batch: the Batch descriptor's, or for a + * single operation, sent as a plain descriptor, that operation's own. */ +static __always_inline volatile uint8_t *batch_status(dto_batch *b) +{ + return b->count == 1 ? &b->comps[0].status : &b->comp.status; +} + +/* Wait out a batch the device may still be writing to. */ +static void batch_quiesce(dto_batch *b) +{ + if (atomic_load_explicit(&b->state, memory_order_acquire) == + BATCH_SUBMITTED) + dsa_wait_no_adjust(batch_status(b)); + while (atomic_load_explicit(&b->state, memory_order_acquire) == + BATCH_REPAIRING) + _mm_pause(); +} + __attribute__((visibility("default"))) -dto_batch_op *dto_batch_op_new(void) +dto_batch *dto_batch_create(int capacity) { - void *p = NULL; + void *b = NULL, *descs = NULL, *comps = NULL; - if (posix_memalign(&p, 64, sizeof(struct dto_batch_op))) + if (capacity < 1) + return NULL; + if (capacity > DTO_BATCH_MAX) + capacity = DTO_BATCH_MAX; + if (posix_memalign(&b, 64, sizeof(struct dto_batch)) || + posix_memalign(&descs, 64, sizeof(struct dsa_hw_desc) * capacity) || + posix_memalign(&comps, 32, + sizeof(struct dsa_completion_record) * capacity)) { + free(b); + free(descs); return NULL; - /* usable before the constructor has run (orig_memset is not) */ - dto_internal_memset(p, 0, sizeof(struct dto_batch_op)); - return (dto_batch_op *)p; + } + /* operations zero their own descriptor and record when added */ + lib_zero(b, sizeof(struct dto_batch)); + ((struct dto_batch *)b)->descs = descs; + ((struct dto_batch *)b)->comps = comps; + ((struct dto_batch *)b)->cap = capacity; + return b; } __attribute__((visibility("default"))) -void dto_batch_op_free(dto_batch_op *op) +void dto_batch_destroy(dto_batch *b) { - free(op); + if (b == NULL) + return; + batch_quiesce(b); + free(b->descs); + free(b->comps); + free(b); } __attribute__((visibility("default"))) -int dto_submit_batch_copy(dto_batch_op *op, void **dst, void **src, - size_t *sizes, int count, unsigned int flags) +void dto_batch_reset(dto_batch *b) +{ + batch_quiesce(b); + b->count = 0; + b->hint = NULL; + atomic_store_explicit(&b->state, BATCH_BUILDING, memory_order_relaxed); +} + +__attribute__((visibility("default"))) +int dto_batch_count(const dto_batch *b) +{ + return b->count; +} + +/* Descriptor for the next operation, or NULL (with *index unset) when the + * batch is full or submitted or n is out of range. BOF is set as requested; + * CC is recorded as requested and settled at submit, once the work queue + * (and so the device's cache-control support) is known. */ +static struct dsa_hw_desc *batch_add(dto_batch *b, uint32_t opcode, + const void *hint, size_t n, + unsigned int flags, bool has_dest, + int *index) +{ + struct dsa_hw_desc *d; + int i; + + if (atomic_load_explicit(&b->state, memory_order_relaxed) != + BATCH_BUILDING || b->count >= b->cap || n == 0 || n > UINT32_MAX) + return NULL; + i = b->count++; + d = &b->descs[i]; + lib_zero(d, sizeof(*d)); + d->opcode = opcode; + d->flags = IDXD_OP_FLAG_CRAV | IDXD_OP_FLAG_RCR; + if (flags & DTO_SUBMIT_BOF) + d->flags |= IDXD_OP_FLAG_BOF; + if ((flags & DTO_SUBMIT_CC) && has_dest) + d->flags |= IDXD_OP_FLAG_CC; + d->completion_addr = (uint64_t)&b->comps[i]; + d->xfer_size = (uint32_t)n; + lib_zero(&b->comps[i], sizeof(b->comps[i])); + if (b->hint == NULL) + b->hint = hint; + *index = i; + return d; +} + +__attribute__((visibility("default"))) +int dto_batch_add_memmove(dto_batch *b, void *dst, const void *src, size_t n, + unsigned int flags) +{ + int i; + struct dsa_hw_desc *d = batch_add(b, DSA_OPCODE_MEMMOVE, dst, n, flags, + true, &i); + + if (d == NULL) + return -1; + d->src_addr = (uint64_t)src; + d->dst_addr = (uint64_t)dst; + return i; +} + +__attribute__((visibility("default"))) +int dto_batch_add_memfill(dto_batch *b, void *dst, uint64_t pattern, size_t n, + unsigned int flags) +{ + int i; + struct dsa_hw_desc *d = batch_add(b, DSA_OPCODE_MEMFILL, dst, n, flags, + true, &i); + + if (d == NULL) + return -1; + d->pattern = pattern; + d->dst_addr = (uint64_t)dst; + return i; +} + +__attribute__((visibility("default"))) +int dto_batch_add_dualcast(dto_batch *b, void *dst1, void *dst2, + const void *src, size_t n, unsigned int flags) +{ + struct dsa_hw_desc *d; + int i; + + /* the device requires both destinations to agree in bits 11:0 */ + if ((((uintptr_t)dst1 ^ (uintptr_t)dst2) & 0xfff) != 0) + return -1; + d = batch_add(b, DSA_OPCODE_DUALCAST, dst1, n, flags, true, &i); + if (d == NULL) + return -1; + d->src_addr = (uint64_t)src; + d->dst_addr = (uint64_t)dst1; + d->dest2 = (uint64_t)dst2; + return i; +} + +__attribute__((visibility("default"))) +int dto_batch_add_crc(dto_batch *b, const void *src, size_t n, + unsigned int flags) +{ + int i; + struct dsa_hw_desc *d = batch_add(b, DSA_OPCODE_CRCGEN, src, n, flags, + false, &i); + + if (d == NULL) + return -1; + d->src_addr = (uint64_t)src; + d->crc_seed = DSA_CRC_SEED_RAW; + return i; +} + +__attribute__((visibility("default"))) +int dto_batch_add_memcpy_crc(dto_batch *b, void *dst, const void *src, + size_t n, unsigned int flags) { + int i; + struct dsa_hw_desc *d = batch_add(b, DSA_OPCODE_COPY_CRC, dst, n, flags, + true, &i); + + if (d == NULL) + return -1; + d->src_addr = (uint64_t)src; + d->dst_addr = (uint64_t)dst; + d->crc_seed = DSA_CRC_SEED_RAW; + return i; +} + +__attribute__((visibility("default"))) +int dto_batch_add_compare(dto_batch *b, const void *src1, const void *src2, + size_t n, unsigned int flags) +{ + int i; + struct dsa_hw_desc *d = batch_add(b, DSA_OPCODE_COMPARE, src1, n, flags, + false, &i); + + if (d == NULL) + return -1; + d->src_addr = (uint64_t)src1; + d->src2_addr = (uint64_t)src2; + return i; +} + +__attribute__((visibility("default"))) +int dto_batch_add_compval(dto_batch *b, const void *src, uint64_t pattern, + size_t n, unsigned int flags) +{ + int i; + struct dsa_hw_desc *d = batch_add(b, DSA_OPCODE_COMPVAL, src, n, flags, + false, &i); + + if (d == NULL) + return -1; + d->src_addr = (uint64_t)src; + d->comp_pattern = pattern; + return i; +} + +__attribute__((visibility("default"))) +int dto_batch_submit(dto_batch *b) +{ + struct dsa_hw_desc *desc; struct dto_wq *wq; - uint32_t desc_flags; - /* a DSA batch needs at least two descriptors */ - if (unlikely(!dto_initialized || use_std_lib_calls || - count < 2 || count > DTO_BATCH_MAX)) + if (atomic_load_explicit(&b->state, memory_order_relaxed) != + BATCH_BUILDING) return DTO_ASYNC_FALLBACK; - wq = get_wq(dst[0]); - if (unlikely(wq == NULL) || (uint32_t)count > wq->max_batch_size) + if (b->count == 0) { + atomic_store_explicit(&b->state, BATCH_DONE, + memory_order_release); + return DTO_ASYNC_SUBMITTED; + } + if (!dto_initialized || use_std_lib_calls) return DTO_ASYNC_FALLBACK; - /* validate everything before touching the op so FALLBACK leaves it - * untouched */ - for (int i = 0; i < count; i++) { - if (sizes[i] == 0 || sizes[i] > wq->max_transfer_size) + wq = get_wq((void *)b->hint); + if (unlikely(wq == NULL) || + (b->count > 1 && (uint32_t)b->count > wq->max_batch_size)) + return DTO_ASYNC_FALLBACK; + for (int i = 0; i < b->count; i++) { + if (b->descs[i].xfer_size > wq->max_transfer_size) return DTO_ASYNC_FALLBACK; } - desc_flags = async_desc_flags(wq, flags, true); - orig_memset(op->descs, 0, sizeof(op->descs[0]) * count); - for (int i = 0; i < count; i++) { - struct dsa_hw_desc *desc = &op->descs[i]; + for (int i = 0; i < b->count; i++) { + struct dsa_hw_desc *d = &b->descs[i]; + + if (!(wq->dsa_gencap & GENCAP_CC_MEMORY)) + d->flags &= ~IDXD_OP_FLAG_CC; + b->comps[i].status = 0; + } + if (b->count == 1) { + desc = &b->descs[0]; + } else { + orig_memset(&b->desc, 0, sizeof(b->desc)); + b->desc.opcode = DSA_OPCODE_BATCH; + b->desc.flags = IDXD_OP_FLAG_CRAV | IDXD_OP_FLAG_RCR; + b->desc.desc_list_addr = (uint64_t)b->descs; + b->desc.desc_count = (uint32_t)b->count; + b->desc.completion_addr = (uint64_t)&b->comp; + b->comp.status = 0; + desc = &b->desc; + } + + if (async_submit(wq, desc, BATCH_SUBMIT_RETRIES) != DTO_ASYNC_SUBMITTED) + return DTO_ASYNC_FALLBACK; + atomic_store_explicit(&b->state, BATCH_SUBMITTED, memory_order_release); + return DTO_ASYNC_SUBMITTED; +} + +/* Fill n bytes with the 8-byte pattern, as Memory Fill does. */ +static void cpu_fill(void *dst, uint64_t pattern, size_t n) +{ + uint8_t *p = dst; + size_t i = 0; + + if (pattern == memfill_pattern((uint8_t)pattern)) { + orig_memset(dst, (uint8_t)pattern, n); + return; + } + for (; i + 8 <= n; i += 8) + *(unaligned_u64 *)(p + i) = pattern; + for (; i < n; i++) + p[i] = (uint8_t)(pattern >> (8 * (i & 7))); +} + +/* Fill in a compare's result the way the device does: result 1 and the + * offset of the first difference, or result 0 with every byte compared. + * With src2 NULL, src1 is compared against the 8-byte pattern instead. */ +static void cpu_compare(struct dsa_completion_record *comp, const void *src1, + const void *src2, uint64_t pattern, size_t n) +{ + const uint8_t *a = src1, *b = src2; + size_t i = 0; - desc->opcode = DSA_OPCODE_MEMMOVE; - desc->flags = desc_flags; - desc->src_addr = (uint64_t)src[i]; - desc->dst_addr = (uint64_t)dst[i]; - desc->xfer_size = (uint32_t)sizes[i]; - desc->completion_addr = (uint64_t)&op->comps[i]; - op->comps[i].status = 0; + /* a word at a time up to the first differing word, then bytewise */ + for (; i + 8 <= n; i += 8) { + uint64_t want = b ? *(const unaligned_u64 *)(b + i) : pattern; + + if (*(const unaligned_u64 *)(a + i) != want) + break; } - orig_memset(&op->desc, 0, sizeof(op->desc)); - op->desc.opcode = DSA_OPCODE_BATCH; - op->desc.flags = IDXD_OP_FLAG_CRAV | IDXD_OP_FLAG_RCR; - op->desc.desc_list_addr = (uint64_t)op->descs; - op->desc.desc_count = count; - op->desc.completion_addr = (uint64_t)&op->comp; - op->comp.status = 0; + for (; i < n; i++) { + uint8_t want = b ? b[i] : (uint8_t)(pattern >> (8 * (i & 7))); - return async_submit(wq, &op->desc); + if (a[i] != want) + break; + } + comp->result = i < n; + comp->bytes_completed = (uint32_t)i; } -__attribute__((visibility("default"))) -int dto_batch_poll(dto_batch_op *op) +/* A move the device stopped partway (a page fault without BOF) cannot be + * redone from the start when the buffers overlap: the device has already + * overwritten part of the source. bytes_completed counts from the end the + * copy started at, which bit 0 of result gives (1: descending). */ +static void cpu_redo_memmove(struct dsa_completion_record *comp, void *dst, + const void *src, size_t n) { - int rc = async_comp_result(&op->comp, DSA_OPCODE_BATCH, - op->desc.desc_count); + size_t done = comp->bytes_completed; - if (rc != DTO_ASYNC_FAILED) - return rc; + if ((comp->status & DSA_COMP_STATUS_MASK) != DSA_COMP_PAGE_FAULT_NOBOF || + done >= n) + done = 0; + if (comp->result & 1) + orig_memmove(dst, src, n - done); + else + orig_memmove((uint8_t *)dst + done, (const uint8_t *)src + done, + n - done); +} + +/* Redo every operation the device did not complete on the CPU. Each status + * is left as the device wrote it, so dto_batch_status() still says who did + * the work; result, bytes_completed and crc_val are filled in as the device + * would have. */ +static void batch_repair(dto_batch *b) +{ + for (int i = 0; i < b->count; i++) { + const struct dsa_hw_desc *d = &b->descs[i]; + struct dsa_completion_record *c = &b->comps[i]; + void *dst = (void *)(uintptr_t)d->dst_addr; + const void *src = (const void *)(uintptr_t)d->src_addr; + size_t n = d->xfer_size; + + if ((c->status & DSA_COMP_STATUS_MASK) == DSA_COMP_SUCCESS) + continue; + switch (d->opcode) { + case DSA_OPCODE_MEMMOVE: + cpu_redo_memmove(c, dst, src, n); + break; + case DSA_OPCODE_MEMFILL: + cpu_fill(dst, d->pattern, n); + break; + case DSA_OPCODE_DUALCAST: + orig_memcpy(dst, src, n); + orig_memcpy((void *)(uintptr_t)d->dest2, src, n); + break; + case DSA_OPCODE_COPY_CRC: + orig_memcpy(dst, src, n); + c->crc_val = cpu_crc32c(d->crc_seed, src, n); + break; + case DSA_OPCODE_CRCGEN: + c->crc_val = cpu_crc32c(d->crc_seed, src, n); + break; + case DSA_OPCODE_COMPARE: + cpu_compare(c, src, (const void *)(uintptr_t)d->src2_addr, + 0, n); + break; + case DSA_OPCODE_COMPVAL: + cpu_compare(c, src, NULL, d->comp_pattern, n); + break; + } + } +} + +__attribute__((visibility("default"))) +int dto_batch_poll(dto_batch *b) +{ + int state = atomic_load_explicit(&b->state, memory_order_acquire); + uint8_t status; - /* redo the copies the device did not complete on the CPU */ - for (uint32_t i = 0; i < op->desc.desc_count; i++) { - const struct dsa_hw_desc *desc = &op->descs[i]; + if (state == BATCH_DONE) + return DTO_ASYNC_DONE; + if (state == BATCH_BUILDING) + return DTO_ASYNC_FAILED; + if (state == BATCH_REPAIRING) + return DTO_ASYNC_PENDING; - if ((op->comps[i].status & DSA_COMP_STATUS_MASK) != - DSA_COMP_SUCCESS) - orig_memcpy((void *)(uintptr_t)desc->dst_addr, - (const void *)(uintptr_t)desc->src_addr, - desc->xfer_size); + status = __atomic_load_n(batch_status(b), __ATOMIC_ACQUIRE); + if (status == 0) + return DTO_ASYNC_PENDING; + if ((status & DSA_COMP_STATUS_MASK) != DSA_COMP_SUCCESS) { + /* exactly one poller repairs; the others see PENDING meanwhile */ + if (!atomic_compare_exchange_strong(&b->state, &state, + BATCH_REPAIRING)) + return state == BATCH_DONE ? DTO_ASYNC_DONE : + DTO_ASYNC_PENDING; + async_log_failure(status, b->count == 1 ? b->descs[0].opcode : + DSA_OPCODE_BATCH, b->count); + batch_repair(b); } - /* Mark the batch complete so a later poll (possibly from another - * thread) doesn't redo the copies from a source the caller may - * already have reused. */ - __atomic_store_n(&op->comp.status, DSA_COMP_SUCCESS, __ATOMIC_RELEASE); + atomic_store_explicit(&b->state, BATCH_DONE, memory_order_release); return DTO_ASYNC_DONE; } + +__attribute__((visibility("default"))) +void dto_batch_wait(dto_batch *b) +{ + if (atomic_load_explicit(&b->state, memory_order_acquire) != + BATCH_SUBMITTED) + return; + dsa_wait_no_adjust(batch_status(b)); + atomic_thread_fence(memory_order_acquire); +} + +__attribute__((visibility("default"))) +int dto_batch_status(const dto_batch *b, int i) +{ + if (i < 0 || i >= b->count) + return 0; + return __atomic_load_n((const uint8_t *)&b->comps[i].status, + __ATOMIC_ACQUIRE) & DSA_COMP_STATUS_MASK; +} + +__attribute__((visibility("default"))) +int dto_batch_result(const dto_batch *b, int i) +{ + if (i < 0 || i >= b->count) + return 0; + return b->comps[i].result; +} + +__attribute__((visibility("default"))) +uint32_t dto_batch_bytes_completed(const dto_batch *b, int i) +{ + if (i < 0 || i >= b->count) + return 0; + return b->comps[i].bytes_completed; +} + +__attribute__((visibility("default"))) +uint32_t dto_batch_crc(const dto_batch *b, int i) +{ + if (i < 0 || i >= b->count) + return 0; + return DSA_CRC_VAL_TO_RAW(b->comps[i].crc_val); +} + +__attribute__((visibility("default"))) +int dto_submit_batch_copy(dto_batch *b, void **dst, void **src, + size_t *sizes, int count, unsigned int flags) +{ + if (count < 1 || count > b->cap) + return DTO_ASYNC_FALLBACK; + dto_batch_reset(b); + for (int i = 0; i < count; i++) { + if (dto_batch_add_memmove(b, dst[i], src[i], sizes[i], + flags) < 0) + return DTO_ASYNC_FALLBACK; + } + return dto_batch_submit(b); +} diff --git a/dto.h b/dto.h index 3f5176e..9fe58fe 100644 --- a/dto.h +++ b/dto.h @@ -7,13 +7,16 @@ * * In addition to transparently accelerating memcpy/memmove/memset/memcmp via * LD_PRELOAD, libdto exposes an explicit submit/poll API for applications - * that want to overlap accelerator data movement with CPU work. + * that want to overlap accelerator work with CPU work. libdto_explicit is + * the same library without the libc interposers, for programs that only call + * this API. * * Lifecycle: * * dto_async_op op; - * if (dto_submit_memcpy(&op, dst, src, n, 1) == DTO_ASYNC_SUBMITTED) { + * if (dto_submit_memcpy(&op, dst, src, n, DTO_SUBMIT_CC) == DTO_ASYNC_SUBMITTED) { * ... other CPU work ... + * dto_async_wait(&op); // optional; or poll in your own loop * while (dto_async_poll(&op) == DTO_ASYNC_PENDING) * ; // pause/yield * if (dto_async_poll(&op) != DTO_ASYNC_DONE) @@ -32,24 +35,49 @@ * Submission returns DTO_ASYNC_FALLBACK when the accelerator is unavailable * (not initialized, disabled via DTO_USESTDC_CALLS, no usable work queue) or * the request is out of range (n == 0, or larger than the work queue's - * maximum transfer size). Nothing is copied in that case: the caller performs + * maximum transfer size). Nothing is done in that case: the caller performs * the operation on the CPU. Unlike the transparent path, explicit submits are * not subject to the size heuristics (DTO_MIN_BYTES / auto-tuning): callers * decide what to offload. * - * On DTO_ASYNC_FAILED (e.g. a page fault the device could not resolve) the - * destination contents are unspecified; redo the whole operation on the CPU. + * On DTO_ASYNC_FAILED the device did not complete the operation; + * dto_async_status() gives its status code. Redo the operation on the CPU. + * After a page fault (status 0x03, see DTO_SUBMIT_BOF) the device has done + * dto_async_bytes_completed() bytes; for a move whose buffers overlap those + * bytes have already overwritten part of the source, so finish the move from + * where the device stopped rather than redoing it: the first bytes were done + * when bit 0 of dto_async_result() is 0 (ascending copy), the last when it is + * 1 (descending). * - * @cache_control (DTO_SUBMIT_CC): directs the operation's output toward the - * CPU cache (IDXD_OP_FLAG_CC) when the device supports it, for destinations - * that will be read again soon. + * Flags (every submit takes a combination of these): * - * DTO_SUBMIT_BOF makes the device block on page faults and wait for the - * kernel to resolve them, instead of aborting the operation (which surfaces - * as DTO_ASYNC_FAILED from dto_async_poll). Use it when the buffers may not - * be fully faulted in. The work queue must be configured with - * block-on-fault enabled ("accel-config config-wq --block-on-fault=1"), - * otherwise the device rejects the descriptor and the operation fails. + * DTO_SUBMIT_CC directs the operation's output toward the CPU cache + * (IDXD_OP_FLAG_CC) when the device supports it, for destinations that will + * be read again soon. Ignored for operations without a destination (CRC + * generation, compare, translation fetch). + * + * DTO_SUBMIT_BOF makes the device block on a page fault until the kernel has + * resolved it, so the operation completes even if its buffers are not + * faulted in. It is passed to the device as given, and the work queue must + * be configured with block-on-fault ("accel-config config-wq + * --block-on-fault=1"): + * + * WQ block-on-fault DTO_SUBMIT_BOF page fault -> + * enabled set the device waits; the op completes + * disabled set (always) DTO_ASYNC_FAILED, status 0x11 + * (invalid flags): the device rejects it + * either not set DTO_ASYNC_FAILED, status 0x03: partial + * completion, see above + * + * The same applies to each operation of a batch, which dto_batch_poll then + * redoes on the CPU. + * + * CRC values: dto_async_crc_val() and dto_batch_crc() return the raw CRC32C + * of the data -- the SSE4.2 crc32 instruction applied from 0, with no final + * inversion. Standard CRC32C (init 0xFFFFFFFF, final inversion) and chained + * checksums are the seeded "crc32c" calls below, in the iSCSI presentation: + * seed 0 gives the standard CRC32C, and passing one call's result as the next + * call's seed continues the checksum across buffers. ******************************************************************************/ #ifndef DTO_H @@ -68,8 +96,8 @@ typedef struct dto_async_op { } dto_async_op; /* Submission flags */ -#define DTO_SUBMIT_CC (1u << 0) /* see @cache_control above */ -#define DTO_SUBMIT_BOF (1u << 1) /* block on page faults (see below) */ +#define DTO_SUBMIT_CC (1u << 0) /* see above */ +#define DTO_SUBMIT_BOF (1u << 1) /* block on page faults (see above) */ /* Submit results */ #define DTO_ASYNC_SUBMITTED 0 @@ -80,7 +108,10 @@ typedef struct dto_async_op { #define DTO_ASYNC_DONE 1 #define DTO_ASYNC_FAILED (-1) -/* Copy n bytes from src to dest. */ +/* ---- Data movement ---- */ + +/* Copy n bytes from src to dest. Overlapping buffers are handled like + * memmove. */ int dto_submit_memcpy(dto_async_op *op, void *dest, const void *src, size_t n, unsigned int flags); @@ -88,49 +119,154 @@ int dto_submit_memcpy(dto_async_op *op, void *dest, const void *src, int dto_submit_memset(dto_async_op *op, void *dest, int c, size_t n, unsigned int flags); -/* Copy n bytes from src to dest and compute the CRC32C of the data. - * The CRC value (read with dto_async_crc_val after DTO_ASYNC_DONE) is the - * standard CRC32C (Castagnoli, seed 0xFFFFFFFF, final inversion), i.e. it - * matches common software crc32c implementations. */ +/* Fill n bytes of dest with the 8-byte pattern, repeated (the low byte of + * pattern lands at dest[0]). */ +int dto_submit_memfill(dto_async_op *op, void *dest, uint64_t pattern, + size_t n, unsigned int flags); + +/* ---- CRC32C ---- */ + +/* Copy n bytes from src to dest and compute the raw CRC32C of the data + * (read it with dto_async_crc_val). */ int dto_submit_memcpy_crc(dto_async_op *op, void *dest, const void *src, size_t n, unsigned int flags); -/* Compute the CRC32C of n bytes at src without copying. */ +/* Raw CRC32C of n bytes at src, without copying (dto_async_crc_val). */ int dto_submit_crc(dto_async_op *op, const void *src, size_t n, unsigned int flags); +/* Seeded CRC32C in the iSCSI presentation (see above), with and without a + * copy. Read the value with dto_async_crc32c_val. */ +int dto_submit_memcpy_crc32c(dto_async_op *op, void *dest, const void *src, + size_t n, uint32_t seed, unsigned int flags); +int dto_submit_crc32c(dto_async_op *op, const void *src, size_t n, + uint32_t seed, unsigned int flags); + +/* Synchronous seeded CRC32C, same presentation as dto_submit_crc32c. Always + * returns the correct value: buffers smaller than DTO_CRC_MIN_BYTES (by + * default DTO_MIN_BYTES) are checksummed on the CPU, as is anything the + * device cannot do. */ +uint32_t dto_crc32c_with_seed(uint32_t seed, const void *src, size_t n); + +/* ---- Compare and translation fetch ---- */ + +/* Compare n bytes at src1 with src2. Once DONE, dto_async_result() is 0 + * when they are equal and 1 when they differ, and dto_async_bytes_completed() + * is then the offset of the first difference. */ +int dto_submit_compare(dto_async_op *op, const void *src1, const void *src2, + size_t n, unsigned int flags); + +/* Compare n bytes at src with the 8-byte pattern, repeated; results as for + * dto_submit_compare. With the buffer's own first 8 bytes as the pattern it + * tells whether the buffer is constant-filled (all zero when they are 0). */ +int dto_submit_compval(dto_async_op *op, const void *src, uint64_t pattern, + size_t n, unsigned int flags); + +/* Translation fetch (DSA 2.0): warm the device's address translations for + * the n bytes at addr without moving data. Devices without the operation + * complete with DTO_ASYNC_FAILED and dto_async_status() == 0x10. */ +int dto_submit_transl_fetch(dto_async_op *op, const void *addr, size_t n, + unsigned int flags); + +/* ---- Completion ---- */ + /* Poll a submitted operation. Returns DTO_ASYNC_PENDING, DTO_ASYNC_DONE or * DTO_ASYNC_FAILED. May be called repeatedly, from any thread; once an op * has left PENDING its result is stable. */ int dto_async_poll(dto_async_op *op); -/* CRC32C result of a completed dto_submit_memcpy_crc/dto_submit_crc. - * Valid only after dto_async_poll returned DTO_ASYNC_DONE. */ -uint32_t dto_async_crc_val(const dto_async_op *op); +/* Wait until the device has finished with a submitted op, using the wait + * method DTO_WAIT_METHOD selects. It only waits: call dto_async_poll + * afterwards for the result. A no-op for an op that was never submitted or + * whose submit returned DTO_ASYNC_FALLBACK, so it is always safe to call + * before a poll loop. There is deliberately no timeout: until the device + * finishes it still owns the buffers, so the only safe thing to do after a + * timeout would be to wait again. */ +void dto_async_wait(dto_async_op *op); + +/* Nonzero when the configured wait method gives the core back to the + * scheduler while waiting, rather than spinning or pausing on it. For callers + * with their own wait loop deciding whether to delegate to dto_async_wait. + * None of the current wait methods does, so this returns 0. */ +int dto_wait_blocks(void); -/* ---- Asynchronous batch copy ---- +/* Results of a completed (DTO_ASYNC_DONE) operation. */ +uint32_t dto_async_crc_val(const dto_async_op *op); /* raw CRC32C */ +uint32_t dto_async_crc32c_val(const dto_async_op *op); /* seeded (iSCSI) */ +int dto_async_result(const dto_async_op *op); /* compare: 0 equal, 1 differ */ +uint32_t dto_async_bytes_completed(const dto_async_op *op); +/* The device's status code (1 = success, 0 = still pending). */ +int dto_async_status(const dto_async_op *op); + +/* ---- Batches ---- + * + * Up to `capacity` (at most DTO_BATCH_MAX) operations sent to the device as + * one batch descriptor: one submission and one completion to poll instead of + * one per operation. Add operations, submit, then poll; dto_batch_reset makes + * the batch reusable. A batch with a single operation is sent as a plain + * descriptor; an empty one completes immediately. The operations of a batch + * may run in any order and concurrently, so none may depend on another. * - * Submits up to DTO_BATCH_MAX copies as one DSA batch descriptor. The caller - * owns the op (dto_batch_op_new / dto_batch_op_free) and may reuse it for - * consecutive batches. A DSA batch requires at least two descriptors, so - * count < 2 returns DTO_ASYNC_FALLBACK, as does count > DTO_BATCH_MAX or - * count > the work queue's configured max_batch_size. + * dto_batch_add_* return the operation's index, or -1 when the batch is full, + * already submitted, or n is 0 (dualcast also requires bits 11:0 of the two + * destinations to be equal, a device rule). dto_batch_submit returns + * DTO_ASYNC_SUBMITTED, or DTO_ASYNC_FALLBACK with nothing submitted (the + * caller does the operations on the CPU), including when the batch is larger + * than the work queue's max_batch_size or an operation exceeds its maximum + * transfer size. * - * dto_submit_batch_copy returns DTO_ASYNC_SUBMITTED or DTO_ASYNC_FALLBACK - * (nothing submitted, nothing copied). dto_batch_poll returns - * DTO_ASYNC_PENDING or DTO_ASYNC_DONE; copies the accelerator failed are - * redone on the CPU before DONE is returned, so DONE means every copy is - * complete. Once DONE, further polls return DONE without touching the - * buffers again. */ -#define DTO_BATCH_MAX 64 + * dto_batch_poll returns DTO_ASYNC_PENDING, then DTO_ASYNC_DONE once every + * operation is complete: operations the device did not finish are redone on + * the CPU before DONE is returned, results included, so after DONE every + * result and CRC is valid. dto_batch_status(b, i) is the device's status for + * operation i (1 when the device did it). A poll of a batch that was never + * submitted, or whose submit fell back, returns DTO_ASYNC_FAILED. + */ +#define DTO_BATCH_MAX 1024 + +typedef struct dto_batch dto_batch; + +/* capacity is clamped to DTO_BATCH_MAX. NULL when out of memory. */ +dto_batch *dto_batch_create(int capacity); +/* Both wait for a batch that is still in flight. */ +void dto_batch_destroy(dto_batch *b); +void dto_batch_reset(dto_batch *b); +int dto_batch_count(const dto_batch *b); + +int dto_batch_add_memmove(dto_batch *b, void *dst, const void *src, size_t n, + unsigned int flags); +int dto_batch_add_memfill(dto_batch *b, void *dst, uint64_t pattern, size_t n, + unsigned int flags); +/* Copy src to both dst1 and dst2. */ +int dto_batch_add_dualcast(dto_batch *b, void *dst1, void *dst2, + const void *src, size_t n, unsigned int flags); +/* Raw CRC32C, read with dto_batch_crc. */ +int dto_batch_add_crc(dto_batch *b, const void *src, size_t n, + unsigned int flags); +int dto_batch_add_memcpy_crc(dto_batch *b, void *dst, const void *src, + size_t n, unsigned int flags); +/* Results as for dto_submit_compare / dto_submit_compval. */ +int dto_batch_add_compare(dto_batch *b, const void *src1, const void *src2, + size_t n, unsigned int flags); +int dto_batch_add_compval(dto_batch *b, const void *src, uint64_t pattern, + size_t n, unsigned int flags); + +int dto_batch_submit(dto_batch *b); +int dto_batch_poll(dto_batch *b); +/* Same contract as dto_async_wait: waits only, poll afterwards. */ +void dto_batch_wait(dto_batch *b); -typedef struct dto_batch_op dto_batch_op; +/* Results of operation i, valid after dto_batch_poll returned DONE. */ +int dto_batch_status(const dto_batch *b, int i); +int dto_batch_result(const dto_batch *b, int i); +uint32_t dto_batch_bytes_completed(const dto_batch *b, int i); +uint32_t dto_batch_crc(const dto_batch *b, int i); -dto_batch_op *dto_batch_op_new(void); -void dto_batch_op_free(dto_batch_op *op); -int dto_submit_batch_copy(dto_batch_op *op, void **dst, void **src, +/* Convenience: reset b and submit count copies (dst[i] <- src[i], sizes[i] + * bytes) as one batch. Returns as dto_batch_submit; DTO_ASYNC_FALLBACK also + * when count is not between 1 and the batch's capacity or a size is 0. */ +int dto_submit_batch_copy(dto_batch *b, void **dst, void **src, size_t *sizes, int count, unsigned int flags); -int dto_batch_poll(dto_batch_op *op); #ifdef __cplusplus } From e9d64eb5bdfe40fb3aba0a8e29242585cad0cb5f Mon Sep 17 00:00:00 2001 From: Daniel Byrne Date: Wed, 21 Oct 2026 08:20:05 -0700 Subject: [PATCH 3/5] Build libdto_explicit, the explicit API without libc interposition libdto_explicit is dto.c built with DTO_NO_LIBC_INTERPOSE, for programs that call the dto_* API directly and must not have every large libc copy in the process sent to DSA (Cloud Hypervisor links it). CMake builds and installs it by default (DTO_BUILD_EXPLICIT) and exports DTO::dto_explicit; the Makefile gains a libdto_explicit target. Also drop the duplicate include(GNUInstallDirs). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Mv7xKtQPkGoXQUvvuuhAEz --- CMakeLists.txt | 78 ++++++++++++++++++++++++++++++++++++-------------- Makefile | 4 +++ 2 files changed, 60 insertions(+), 22 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 102964d..8e8cccd 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -23,18 +23,34 @@ option(DTO_NUMA_SUPPORT "Enable NUMA awareness support (requires libnuma)" ON) option(DTO_STATIC_ACCEL_CONFIG "Statically link accel-config into libdto.so" OFF) option(DTO_STATIC_NUMA "Statically link numa into libdto.so (requires PIC-compiled libnuma.a)" OFF) +# libdto_explicit is the same library without the memcpy/memset/memmove/ +# memcmp interposers, for programs that call the dto_* API (dto.h) directly +# and must not have every large libc copy in the process sent to DSA. +option(DTO_BUILD_EXPLICIT "Also build libdto_explicit, without libc interposition" ON) + # Build the shared library add_library(dto SHARED dto.c) add_library(DTO::dto ALIAS dto) -target_include_directories(dto - PUBLIC - $ - $) +set(DTO_LIBRARIES dto) +if(DTO_BUILD_EXPLICIT) + add_library(dto_explicit SHARED dto.c) + add_library(DTO::dto_explicit ALIAS dto_explicit) + target_compile_definitions(dto_explicit PRIVATE DTO_NO_LIBC_INTERPOSE) + list(APPEND DTO_LIBRARIES dto_explicit) +endif() + +foreach(lib ${DTO_LIBRARIES}) + target_include_directories(${lib} + PUBLIC + $ + $) +endforeach() # set gnu source everywhere add_compile_definitions(_GNU_SOURCE) -# Add the -DDTO_STATS_SUPPORT preprocessor definition +# Add the -DDTO_STATS_SUPPORT preprocessor definition. Stats cover the +# interposed calls, so libdto_explicit has none to collect. target_compile_definitions(dto PRIVATE DTO_STATS_SUPPORT) # Link libraries @@ -43,7 +59,9 @@ set(DTO_DYNAMIC_LIBS dl) # Handle accel-config support if(DTO_ACCEL_CONFIG_SUPPORT) - target_compile_definitions(dto PRIVATE DTO_ACCEL_CONFIG_SUPPORT) + foreach(lib ${DTO_LIBRARIES}) + target_compile_definitions(${lib} PRIVATE DTO_ACCEL_CONFIG_SUPPORT) + endforeach() message(STATUS "accel-config support: enabled") if(DTO_STATIC_ACCEL_CONFIG) @@ -62,7 +80,9 @@ endif() # Handle numa support if(DTO_NUMA_SUPPORT) - target_compile_definitions(dto PRIVATE DTO_NUMA_SUPPORT) + foreach(lib ${DTO_LIBRARIES}) + target_compile_definitions(${lib} PRIVATE DTO_NUMA_SUPPORT) + endforeach() message(STATUS "NUMA support: enabled") if(DTO_STATIC_NUMA) @@ -83,21 +103,25 @@ endif() # Link everything together. Use PRIVATE so accel-config/numa/dl are the shared # library's own dependencies (resolved via its DT_NEEDED) rather than INTERFACE # requirements that would be forced onto every downstream consumer of DTO::dto. -if(DTO_STATIC_LIBS) - target_link_libraries(dto PRIVATE - -Wl,--whole-archive - ${DTO_STATIC_LIBS} - -Wl,--no-whole-archive - ${DTO_DYNAMIC_LIBS} - ) -else() - target_link_libraries(dto PRIVATE ${DTO_DYNAMIC_LIBS}) -endif() +foreach(lib ${DTO_LIBRARIES}) + if(DTO_STATIC_LIBS) + target_link_libraries(${lib} PRIVATE + -Wl,--whole-archive + ${DTO_STATIC_LIBS} + -Wl,--no-whole-archive + ${DTO_DYNAMIC_LIBS} + ) + else() + target_link_libraries(${lib} PRIVATE ${DTO_DYNAMIC_LIBS}) + endif() +endforeach() include(CheckCCompilerFlag) check_c_compiler_flag("-mwaitpkg" HAS_WAITPKG) if (HAS_WAITPKG) - target_compile_options(dto PRIVATE -mwaitpkg -march=native) + foreach(lib ${DTO_LIBRARIES}) + target_compile_options(${lib} PRIVATE -mwaitpkg -march=native) + endforeach() endif() # Build dto-test and dto-test-wodto @@ -123,6 +147,18 @@ if(DTO_BUILD_TESTS) TIMEOUT 120 ) + # The same tests against libdto_explicit: the explicit API without the + # interposers (the libc mem* tests then exercise libc itself). + if(DTO_BUILD_EXPLICIT) + add_executable(dto-test-functional-explicit tests/test_functional.c) + target_link_libraries(dto-test-functional-explicit PRIVATE DTO::dto_explicit pthread) + add_test(NAME functional_explicit COMMAND dto-test-functional-explicit) + set_tests_properties(functional_explicit PROPERTIES + LABELS "functional" + TIMEOUT 120 + ) + endif() + # ---- A/B performance regression test ---- # The baseline object is compiled from unguarded upstream sources that # always reference accel-config and numa, so the perf test can only be @@ -225,12 +261,10 @@ if(DTO_BUILD_TESTS) endif() endif() -include(GNUInstallDirs) - install(FILES dto.h DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}) -# Install and export the library -install(TARGETS dto +# Install and export the libraries +install(TARGETS ${DTO_LIBRARIES} EXPORT DTOTargets RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} diff --git a/Makefile b/Makefile index d0223b5..4da8111 100644 --- a/Makefile +++ b/Makefile @@ -12,6 +12,10 @@ libdto: dto.c libdto_nostats: dto.c gcc -shared -fPIC -Wl,-soname,libdto.so dto.c $(DML_LIB_CXX) -DDTO_ACCEL_CONFIG_SUPPORT -DDTO_NUMA_SUPPORT -o libdto.so.1.0 -laccel-config -ldl -lnuma -mwaitpkg +# the explicit dto_* API without the libc memcpy/memset/memmove/memcmp interposers +libdto_explicit: dto.c + gcc -shared -fPIC -Wl,-soname,libdto_explicit.so dto.c $(DML_LIB_CXX) -DDTO_NO_LIBC_INTERPOSE -DDTO_ACCEL_CONFIG_SUPPORT -DDTO_NUMA_SUPPORT -o libdto_explicit.so.1.0 -laccel-config -ldl -lnuma -mwaitpkg + install: cp libdto.so.1.0 /usr/lib64/ cp dto.h /usr/include/ From 61c5ff4e098354b16f668069225d4bf85098c5e1 Mon Sep 17 00:00:00 2001 From: Daniel Byrne Date: Wed, 21 Oct 2026 08:20:06 -0700 Subject: [PATCH 4/5] tests: cover the unified async API, and require offload on DSA Test every submit and batch operation against CPU results: raw and seeded CRCs (including chaining and the CRC32C check value), compare and compval offsets, memfill patterns, translation fetch, waits on unsubmitted ops, batch limits, the single-descriptor path, and the CPU repair of a batch whose operations hit unmapped pages, including overlapping moves the device left half done in both directions and two threads polling the same batch. test_async_bof checks the block-on-fault contract on whatever the work queue is configured with: a BOF copy into untouched pages completes on a block-on-fault queue and is rejected (invalid flags) on one without; without the flag the copy stops at the first untouched page with the touched part done; and batch operations follow the same rules. The tests accepted DTO_ASYNC_FAILED and FALLBACK everywhere, so they passed while every DTO_SUBMIT_BOF operation failed on a work queue without block-on-fault. With DTO_TEST_EXPECT_DSA set they now require operations on faulted-in buffers to be submitted and completed by the device, and the general tests no longer request block-on-fault, so they hold on any work queue configuration. The suite also runs against libdto_explicit (functional_explicit). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Mv7xKtQPkGoXQUvvuuhAEz --- tests/test_functional.c | 717 ++++++++++++++++++++++++++++++++++++---- 1 file changed, 649 insertions(+), 68 deletions(-) diff --git a/tests/test_functional.c b/tests/test_functional.c index eb667fd..c49aaba 100644 --- a/tests/test_functional.c +++ b/tests/test_functional.c @@ -23,6 +23,7 @@ #include #include #include +#include /* Buffer sizes spanning below and above the DSA threshold (65536) */ static const size_t test_sizes[] = { @@ -509,18 +510,37 @@ static int test_multithread(void) /* ---- Test runner ---- */ -/* ---- Explicit asynchronous API (dto.h) ---- */ +/* ---- Explicit asynchronous API (dto.h) ---- + * + * Without DSA every submit falls back and the tests check the documented + * fallback paths. With DTO_TEST_EXPECT_DSA set (a usable work queue + * configured), operations on faulted-in buffers must be submitted and must + * complete on the device, so a regression that quietly pushes work back onto + * the CPU fails instead of passing. + */ + +static int expect_dsa; -/* Standard CRC32C (Castagnoli): init 0xFFFFFFFF, final inversion, the - * convention dto.h documents for dto_async_crc_val. */ +/* CRC32C loop over the SSE4.2 instruction: crc32c_loop(0, ...) is the raw + * presentation and ~crc32c_loop(~seed, ...) the seeded (iSCSI) one. */ __attribute__((target("sse4.2"))) -static uint32_t ref_crc32c(const void *buf, size_t n) +static uint32_t crc32c_loop(uint32_t crc, const void *buf, size_t n) { const uint8_t *p = buf; - uint32_t crc = 0xFFFFFFFFu; + for (size_t i = 0; i < n; i++) crc = _mm_crc32_u8(crc, p[i]); - return ~crc; + return crc; +} + +static uint32_t ref_crc_raw(const void *buf, size_t n) +{ + return crc32c_loop(0, buf, n); +} + +static uint32_t ref_crc32c(uint32_t seed, const void *buf, size_t n) +{ + return ~crc32c_loop(~seed, buf, n); } /* Poll until poll_expr leaves PENDING and yield its result. A submitted op @@ -541,6 +561,29 @@ static uint32_t ref_crc32c(const void *buf, size_t n) } \ _rc; }) +/* The outcome of a submit, then of its poll: SUBMITTED and DONE when DSA is + * expected, any documented outcome otherwise. */ +#define CHECK_SUBMIT(rc) do { \ + if (expect_dsa) \ + ASSERT_EQ((rc), DTO_ASYNC_SUBMITTED); \ + else \ + ASSERT_TRUE((rc) == DTO_ASYNC_SUBMITTED || (rc) == DTO_ASYNC_FALLBACK); \ +} while (0) + +#define CHECK_DONE(rc) do { \ + if (expect_dsa) \ + ASSERT_EQ((rc), DTO_ASYNC_DONE); \ + else \ + ASSERT_TRUE((rc) == DTO_ASYNC_DONE || (rc) == DTO_ASYNC_FAILED); \ +} while (0) + +/* Wait for a submitted op the documented way: wait, then poll. */ +static int finish(dto_async_op *op) +{ + dto_async_wait(op); + return ASYNC_WAIT(dto_async_poll(op)); +} + /* Submit a memcpy with the given flags and verify the data afterwards, * whichever of the three documented outcomes (DONE, FAILED + CPU redo, * FALLBACK + CPU copy) occurs. */ @@ -556,13 +599,13 @@ static int check_async_memcpy(unsigned int flags) clear_buf(dst, n); rc = dto_submit_memcpy(&op, dst, src, n, flags); + CHECK_SUBMIT(rc); if (rc == DTO_ASYNC_SUBMITTED) { - rc = ASYNC_WAIT(dto_async_poll(&op)); - ASSERT_TRUE(rc == DTO_ASYNC_DONE || rc == DTO_ASYNC_FAILED); + rc = finish(&op); + CHECK_DONE(rc); if (rc != DTO_ASYNC_DONE) memcpy(dst, src, n); /* documented caller fallback */ } else { - ASSERT_EQ(rc, DTO_ASYNC_FALLBACK); memcpy(dst, src, n); } ASSERT_TRUE(verify_equal(dst, src, n)); @@ -571,22 +614,21 @@ static int check_async_memcpy(unsigned int flags) return 0; } +/* The general tests use faulted-in buffers and no DTO_SUBMIT_BOF, so they + * complete on any work queue; test_async_bof covers block-on-fault. */ static int test_async_memcpy(void) { - return check_async_memcpy(DTO_SUBMIT_CC | DTO_SUBMIT_BOF); + return check_async_memcpy(DTO_SUBMIT_CC); } -static int test_async_memcpy_no_bof(void) +static int test_async_memcpy_no_cc(void) { - /* Without DTO_SUBMIT_BOF a page fault aborts the operation and poll - * reports DTO_ASYNC_FAILED; the caller redoes the copy on the CPU. - * Either outcome is valid here, the data must be right afterwards. */ return check_async_memcpy(0); } static int test_async_memset(void) { - const size_t n = 1 << 20; + const size_t n = (1 << 20) + 3; uint8_t *dst = malloc(n); dto_async_op op; int rc; @@ -594,44 +636,61 @@ static int test_async_memset(void) ASSERT_TRUE(dst != NULL); clear_buf(dst, n); - rc = dto_submit_memset(&op, dst, 0x5A, n, DTO_SUBMIT_BOF); + rc = dto_submit_memset(&op, dst, 0x5A, n, DTO_SUBMIT_CC); + CHECK_SUBMIT(rc); if (rc == DTO_ASYNC_SUBMITTED) { - rc = ASYNC_WAIT(dto_async_poll(&op)); - ASSERT_TRUE(rc == DTO_ASYNC_DONE || rc == DTO_ASYNC_FAILED); - if (rc != DTO_ASYNC_DONE) - memset(dst, 0x5A, n); - } else { - ASSERT_EQ(rc, DTO_ASYNC_FALLBACK); - memset(dst, 0x5A, n); + rc = finish(&op); + CHECK_DONE(rc); } + if (rc != DTO_ASYNC_DONE) + memset(dst, 0x5A, n); ASSERT_TRUE(verify_set(dst, 0x5A, n)); free(dst); return 0; } -/* BOF is rejected by the device on a WQ configured without block-on-fault - * (dto.h), so like the other tests the CRC tests accept DTO_ASYNC_FAILED; - * the CRC value is only checked when the device completed the op. */ +static int test_async_memfill(void) +{ + const uint64_t pattern = 0x0123456789ABCDEFULL; + const size_t n = (256 * 1024) + 5; /* not a multiple of 8 */ + uint8_t *dst = malloc(n); + dto_async_op op; + int rc; + + ASSERT_TRUE(dst != NULL); + clear_buf(dst, n); + + rc = dto_submit_memfill(&op, dst, pattern, n, DTO_SUBMIT_CC); + CHECK_SUBMIT(rc); + if (rc != DTO_ASYNC_SUBMITTED) + return 0; + rc = finish(&op); + CHECK_DONE(rc); + if (rc == DTO_ASYNC_DONE) { + for (size_t i = 0; i < n; i++) + ASSERT_EQ(dst[i], (uint8_t)(pattern >> (8 * (i & 7)))); + } + free(dst); + return 0; +} + static int test_async_crc(void) { - const size_t n = 256 * 1024; + const size_t n = 256 * 1024 + 3; uint8_t *src = malloc(n); dto_async_op op; - uint32_t expect; int rc; ASSERT_TRUE(src != NULL); fill_pattern(src, n); - expect = ref_crc32c(src, n); - rc = dto_submit_crc(&op, src, n, DTO_SUBMIT_BOF); + rc = dto_submit_crc(&op, src, n, 0); + CHECK_SUBMIT(rc); if (rc == DTO_ASYNC_SUBMITTED) { - rc = ASYNC_WAIT(dto_async_poll(&op)); - ASSERT_TRUE(rc == DTO_ASYNC_DONE || rc == DTO_ASYNC_FAILED); + rc = finish(&op); + CHECK_DONE(rc); if (rc == DTO_ASYNC_DONE) - ASSERT_EQ(dto_async_crc_val(&op), expect); - } else { - ASSERT_EQ(rc, DTO_ASYNC_FALLBACK); + ASSERT_EQ(dto_async_crc_val(&op), ref_crc_raw(src, n)); } free(src); return 0; @@ -642,50 +701,557 @@ static int test_async_memcpy_crc(void) const size_t n = 256 * 1024; uint8_t *src = malloc(n), *dst = malloc(n); dto_async_op op; - uint32_t expect; int rc; ASSERT_TRUE(src && dst); fill_pattern(src, n); clear_buf(dst, n); - expect = ref_crc32c(src, n); - rc = dto_submit_memcpy_crc(&op, dst, src, n, DTO_SUBMIT_CC | DTO_SUBMIT_BOF); + rc = dto_submit_memcpy_crc(&op, dst, src, n, DTO_SUBMIT_CC); + CHECK_SUBMIT(rc); if (rc == DTO_ASYNC_SUBMITTED) { - rc = ASYNC_WAIT(dto_async_poll(&op)); - ASSERT_TRUE(rc == DTO_ASYNC_DONE || rc == DTO_ASYNC_FAILED); - if (rc == DTO_ASYNC_DONE) - ASSERT_EQ(dto_async_crc_val(&op), expect); - else - memcpy(dst, src, n); - } else { - ASSERT_EQ(rc, DTO_ASYNC_FALLBACK); - memcpy(dst, src, n); + rc = finish(&op); + CHECK_DONE(rc); } + if (rc == DTO_ASYNC_DONE) + ASSERT_EQ(dto_async_crc_val(&op), ref_crc_raw(src, n)); + else + memcpy(dst, src, n); ASSERT_TRUE(verify_equal(dst, src, n)); free(src); free(dst); return 0; } +/* Seeded (iSCSI) CRC32C: seed 0 is the standard CRC32C, and chaining two + * halves gives the CRC of the whole. */ +static int test_async_crc32c(void) +{ + const size_t n = 128 * 1024, half = n / 2; + uint8_t *src = malloc(n), *dst = malloc(n); + uint32_t first; + dto_async_op op; + int rc; + + ASSERT_TRUE(src && dst); + fill_pattern(src, n); + clear_buf(dst, n); + + /* the reference itself: "123456789" is the CRC32C check string */ + ASSERT_EQ(ref_crc32c(0, "123456789", 9), 0xE3069283u); + + rc = dto_submit_crc32c(&op, src, half, 0, 0); + CHECK_SUBMIT(rc); + if (rc == DTO_ASYNC_SUBMITTED) { + rc = finish(&op); + CHECK_DONE(rc); + if (rc == DTO_ASYNC_DONE) { + first = dto_async_crc32c_val(&op); + ASSERT_EQ(first, ref_crc32c(0, src, half)); + rc = dto_submit_memcpy_crc32c(&op, dst + half, src + half, + n - half, first, DTO_SUBMIT_CC); + CHECK_SUBMIT(rc); + if (rc == DTO_ASYNC_SUBMITTED && finish(&op) == DTO_ASYNC_DONE) { + ASSERT_EQ(dto_async_crc32c_val(&op), + ref_crc32c(0, src, n)); + ASSERT_TRUE(verify_equal(dst + half, src + half, n - half)); + } + } + } + + /* the synchronous call is always right: CPU below DTO_CRC_MIN_BYTES, + * DSA above, and the seed alone for an empty buffer */ + ASSERT_EQ(dto_crc32c_with_seed(0, src, 100), ref_crc32c(0, src, 100)); + ASSERT_EQ(dto_crc32c_with_seed(0, src, n), ref_crc32c(0, src, n)); + ASSERT_EQ(dto_crc32c_with_seed(dto_crc32c_with_seed(0, src, half), + src + half, n - half), + ref_crc32c(0, src, n)); + ASSERT_EQ(dto_crc32c_with_seed(0x12345678u, src, 0), 0x12345678u); + free(src); + free(dst); + return 0; +} + +/* Offset of the first difference, as compare reports it */ +static int check_compare(const uint8_t *a, const uint8_t *b, size_t n, + int expect_result, uint32_t expect_offset) +{ + dto_async_op op; + int rc = dto_submit_compare(&op, a, b, n, 0); + + CHECK_SUBMIT(rc); + if (rc != DTO_ASYNC_SUBMITTED) + return 0; + rc = finish(&op); + CHECK_DONE(rc); + if (rc == DTO_ASYNC_DONE) { + ASSERT_EQ(dto_async_result(&op), expect_result); + if (expect_result) + ASSERT_EQ(dto_async_bytes_completed(&op), expect_offset); + } + return 0; +} + +static int test_async_compare(void) +{ + static const size_t offsets[] = { 0, 1, 7, 8, 63, 4095, 4096, 100003 }; + const size_t n = 128 * 1024; + uint8_t *a = malloc(n), *b = malloc(n); + + ASSERT_TRUE(a && b); + fill_pattern(a, n); + fill_pattern(b, n); + if (check_compare(a, b, n, 0, 0)) + return 1; + for (size_t i = 0; i < sizeof(offsets) / sizeof(offsets[0]); i++) { + b[offsets[i]] ^= 0x40; + if (check_compare(a, b, n, 1, offsets[i])) + return 1; + b[offsets[i]] ^= 0x40; + } + b[n - 1] ^= 1; + if (check_compare(a, b, n, 1, n - 1)) + return 1; + free(a); + free(b); + return 0; +} + +static int test_async_compval(void) +{ + const uint64_t pattern = 0xCCCCCCCC00000000ULL | 0x5A5A0F0F; + const size_t n = 64 * 1024; + uint8_t *buf = malloc(n); + dto_async_op op; + int rc; + + ASSERT_TRUE(buf != NULL); + for (size_t i = 0; i < n; i++) + ((volatile uint8_t *)buf)[i] = (uint8_t)(pattern >> (8 * (i & 7))); + + for (int round = 0; round < 2; round++) { + /* round 0: buf is the pattern; round 1: one byte differs */ + const size_t off = 12345; + + if (round == 1) + buf[off] ^= 0xFF; + rc = dto_submit_compval(&op, buf, pattern, n, 0); + CHECK_SUBMIT(rc); + if (rc != DTO_ASYNC_SUBMITTED) + continue; + rc = finish(&op); + CHECK_DONE(rc); + if (rc != DTO_ASYNC_DONE) + continue; + ASSERT_EQ(dto_async_result(&op), round); + if (round == 1) + ASSERT_EQ(dto_async_bytes_completed(&op), off); + } + free(buf); + return 0; +} + +static int test_async_transl_fetch(void) +{ + const size_t n = 1 << 20; + uint8_t *buf = malloc(n); + dto_async_op op; + int rc; + + ASSERT_TRUE(buf != NULL); + clear_buf(buf, n); + rc = dto_submit_transl_fetch(&op, buf, n, 0); + CHECK_SUBMIT(rc); + if (rc == DTO_ASYNC_SUBMITTED) { + rc = finish(&op); + /* devices without the operation report BAD_OPCODE */ + if (rc != DTO_ASYNC_DONE) { + ASSERT_EQ(rc, DTO_ASYNC_FAILED); + ASSERT_EQ(dto_async_status(&op), 0x10); + } + } + free(buf); + return 0; +} + +/* DTO_SUBMIT_BOF is passed to the device as given (dto.h). Learn whether the + * work queue has block-on-fault from a BOF submit on faulted-in memory, then + * copy into a destination whose first pages are touched and the rest not: + * BOF set, WQ with block-on-fault: the device waits; the copy completes + * BOF set, WQ without block-on-fault: rejected, status 0x11 (invalid flags) + * BOF not set, any WQ: stops at the first untouched page, + * status 0x03, the touched part done + * Assumes every work queue DTO uses has the same block-on-fault setting. */ +static int test_async_bof(void) +{ + const size_t page = 4096, n = 16 * page, warm = 4 * page; + uint8_t *src = malloc(n), *hot = malloc(n); + dto_batch *b = dto_batch_create(2); + dto_async_op op; + int rc, wq_bof; + + ASSERT_TRUE(src && hot && b); + fill_pattern(src, n); + clear_buf(hot, n); + + rc = dto_submit_memcpy(&op, hot, src, n, DTO_SUBMIT_BOF); + CHECK_SUBMIT(rc); + if (rc != DTO_ASYNC_SUBMITTED) + return 0; + rc = finish(&op); + if (rc == DTO_ASYNC_DONE) { + wq_bof = 1; + ASSERT_TRUE(verify_equal(hot, src, n)); + } else { + ASSERT_EQ(rc, DTO_ASYNC_FAILED); + ASSERT_EQ(dto_async_status(&op), 0x11); + wq_bof = 0; + } + printf(" (work queue block-on-fault: %s)\n", wq_bof ? "enabled" : "disabled"); + + for (int bof = 0; bof < 2; bof++) { + uint8_t *dst = mmap(NULL, n, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + + ASSERT_TRUE(dst != MAP_FAILED); + clear_buf(dst, warm); /* touch the first pages only */ + rc = dto_submit_memcpy(&op, dst, src, n, bof ? DTO_SUBMIT_BOF : 0); + CHECK_SUBMIT(rc); + if (rc == DTO_ASYNC_SUBMITTED) { + rc = finish(&op); + if (bof && wq_bof) { + ASSERT_EQ(rc, DTO_ASYNC_DONE); + ASSERT_TRUE(verify_equal(dst, src, n)); + } else if (bof) { + ASSERT_EQ(rc, DTO_ASYNC_FAILED); + ASSERT_EQ(dto_async_status(&op), 0x11); + } else { + ASSERT_EQ(rc, DTO_ASYNC_FAILED); + ASSERT_EQ(dto_async_status(&op), 0x03); + ASSERT_EQ(dto_async_bytes_completed(&op), warm); + ASSERT_TRUE(verify_equal(dst, src, warm)); + } + } + munmap(dst, n); + } + + /* the same per batch operation; dto_batch_poll redoes the rejected ones */ + clear_buf(hot, n); + ASSERT_EQ(dto_batch_add_memmove(b, hot, src, n / 2, DTO_SUBMIT_BOF), 0); + ASSERT_EQ(dto_batch_add_memmove(b, hot + n / 2, src + n / 2, n / 2, 0), 1); + rc = dto_batch_submit(b); + CHECK_SUBMIT(rc); + if (rc == DTO_ASYNC_SUBMITTED) { + ASSERT_EQ(ASYNC_WAIT(dto_batch_poll(b)), DTO_ASYNC_DONE); + ASSERT_EQ(dto_batch_status(b, 0), wq_bof ? 1 : 0x11); + ASSERT_EQ(dto_batch_status(b, 1), 1); + ASSERT_TRUE(verify_equal(hot, src, n)); + } + dto_batch_destroy(b); + free(src); + free(hot); + return 0; +} + +/* dto_async_wait must return at once for ops that were never submitted or + * fell back: their status byte will never be written. */ +static int test_async_wait_unsubmitted(void) +{ + dto_async_op op; + uint8_t a[64], b[64]; + + memset(&op, 0xAB, sizeof(op)); + dto_async_wait(&op); + ASSERT_EQ(dto_submit_memcpy(&op, a, b, 0, 0), DTO_ASYNC_FALLBACK); + dto_async_wait(&op); + ASSERT_EQ(dto_wait_blocks(), 0); + return 0; +} + static int test_async_bad_args(void) { uint8_t a[64], b[64]; - void *dstv[1] = { a }, *srcv[1] = { b }; - size_t sizes[1] = { 64 }; + struct { char pad[16]; dto_async_op op; } __attribute__((aligned(64))) s; + dto_async_op *unaligned = (dto_async_op *)&s.pad[8]; dto_async_op op; - dto_batch_op *bop; /* zero length never reaches the device */ ASSERT_EQ(dto_submit_memcpy(&op, a, b, 0, 0), DTO_ASYNC_FALLBACK); + ASSERT_EQ(dto_submit_memset(&op, a, 0, 0, 0), DTO_ASYNC_FALLBACK); + ASSERT_EQ(dto_submit_memfill(&op, a, 0, 0, 0), DTO_ASYNC_FALLBACK); ASSERT_EQ(dto_submit_crc(&op, b, 0, 0), DTO_ASYNC_FALLBACK); - - /* a DSA batch needs at least two descriptors */ - bop = dto_batch_op_new(); - ASSERT_TRUE(bop != NULL); - ASSERT_EQ(dto_submit_batch_copy(bop, dstv, srcv, sizes, 1, 0), + ASSERT_EQ(dto_submit_crc32c(&op, b, 0, 0, 0), DTO_ASYNC_FALLBACK); + ASSERT_EQ(dto_submit_memcpy_crc(&op, a, b, 0, 0), DTO_ASYNC_FALLBACK); + ASSERT_EQ(dto_submit_compare(&op, a, b, 0, 0), DTO_ASYNC_FALLBACK); + ASSERT_EQ(dto_submit_compval(&op, a, 0, 0, 0), DTO_ASYNC_FALLBACK); + ASSERT_EQ(dto_submit_transl_fetch(&op, a, 0, 0), DTO_ASYNC_FALLBACK); + /* the completion record needs 32-byte alignment */ + ASSERT_EQ(dto_submit_memcpy(unaligned, a, b, sizeof(a), 0), DTO_ASYNC_FALLBACK); - dto_batch_op_free(bop); + return 0; +} + +/* ---- Batches ---- */ + +static int test_batch_limits(void) +{ + dto_batch *b = dto_batch_create(DTO_BATCH_MAX + 5); + uint8_t *buf = aligned_alloc(4096, 3 * 4096); + int i; + + ASSERT_TRUE(b && buf); + clear_buf(buf, 3 * 4096); + /* polling a batch that was never submitted reports FAILED */ + ASSERT_EQ(dto_batch_poll(b), DTO_ASYNC_FAILED); + /* an empty batch completes at once */ + ASSERT_EQ(dto_batch_submit(b), DTO_ASYNC_SUBMITTED); + ASSERT_EQ(dto_batch_poll(b), DTO_ASYNC_DONE); + /* a submitted batch takes no more operations until reset */ + ASSERT_EQ(dto_batch_add_memfill(b, buf, 0, 64, 0), -1); + dto_batch_reset(b); + + ASSERT_EQ(dto_batch_add_memmove(b, buf, buf + 64, 0, 0), -1); + /* dualcast destinations must agree in bits 11:0 */ + ASSERT_EQ(dto_batch_add_dualcast(b, buf + 4096, buf + 2 * 4096 + 64, + buf, 64, 0), -1); + /* the capacity is clamped to DTO_BATCH_MAX */ + for (i = 0; i < DTO_BATCH_MAX; i++) + ASSERT_EQ(dto_batch_add_compval(b, buf, 0, 64, 0), i); + ASSERT_EQ(dto_batch_add_compval(b, buf, 0, 64, 0), -1); + ASSERT_EQ(dto_batch_count(b), DTO_BATCH_MAX); + dto_batch_reset(b); + ASSERT_EQ(dto_batch_count(b), 0); + dto_batch_destroy(b); + dto_batch_destroy(NULL); + free(buf); + return 0; +} + +/* Every operation kind in one batch, checked against CPU results. */ +static int run_batch_all_ops(int nops_limit) +{ + const size_t n = 32 * 1024 + 8; + const uint64_t pattern = 0x1122334455667788ULL; + uint8_t *src = aligned_alloc(4096, n + 4096); + uint8_t *src2 = aligned_alloc(4096, n + 4096); + uint8_t *mv = aligned_alloc(4096, n + 4096); + uint8_t *fill = aligned_alloc(4096, n + 4096); + uint8_t *d1 = aligned_alloc(4096, n + 4096); + uint8_t *d2 = aligned_alloc(4096, n + 4096); + uint8_t *cc = aligned_alloc(4096, n + 4096); + uint8_t *pat = aligned_alloc(4096, n + 4096); + dto_batch *b = dto_batch_create(16); + int ops[7], count = 0, rc; + + ASSERT_TRUE(src && src2 && mv && fill && d1 && d2 && cc && pat && b); + /* operations in one batch may run concurrently, so none of them reads + * what another writes */ + for (size_t i = 0; i < n; i++) + ((volatile uint8_t *)pat)[i] = (uint8_t)(pattern >> (8 * (i & 7))); + fill_pattern(src, n); + fill_pattern(src2, n); + src2[777] ^= 1; + clear_buf(mv, n); + clear_buf(fill, n); + clear_buf(d1, n); + clear_buf(d2, n); + clear_buf(cc, n); + + ops[0] = dto_batch_add_memmove(b, mv, src, n, DTO_SUBMIT_CC); + ops[1] = dto_batch_add_memfill(b, fill, pattern, n, 0); + ops[2] = dto_batch_add_dualcast(b, d1, d2, src, n, 0); + ops[3] = dto_batch_add_crc(b, src, n, 0); + ops[4] = dto_batch_add_memcpy_crc(b, cc, src, n, DTO_SUBMIT_CC); + ops[5] = dto_batch_add_compare(b, src, src2, n, 0); + ops[6] = dto_batch_add_compval(b, pat, pattern, n, 0); + for (int i = 0; i < 7; i++) + ASSERT_EQ(ops[i], i); + count = 7; + /* nops_limit 1 exercises the plain-descriptor path */ + if (nops_limit == 1) { + dto_batch_reset(b); + ASSERT_EQ(dto_batch_add_memcpy_crc(b, cc, src, n, 0), 0); + count = 1; + } + + rc = dto_batch_submit(b); + CHECK_SUBMIT(rc); + if (rc != DTO_ASYNC_SUBMITTED) + return 0; + dto_batch_wait(b); + ASSERT_EQ(ASYNC_WAIT(dto_batch_poll(b)), DTO_ASYNC_DONE); + ASSERT_EQ(dto_batch_poll(b), DTO_ASYNC_DONE); /* stays DONE */ + for (int i = 0; i < count; i++) { + if (expect_dsa) + ASSERT_EQ(dto_batch_status(b, i), 1); + } + + if (count == 1) { + ASSERT_TRUE(verify_equal(cc, src, n)); + ASSERT_EQ(dto_batch_crc(b, 0), ref_crc_raw(src, n)); + } else { + ASSERT_TRUE(verify_equal(mv, src, n)); + for (size_t i = 0; i < n; i++) + ASSERT_EQ(fill[i], (uint8_t)(pattern >> (8 * (i & 7)))); + ASSERT_TRUE(verify_equal(d1, src, n)); + ASSERT_TRUE(verify_equal(d2, src, n)); + ASSERT_EQ(dto_batch_crc(b, 3), ref_crc_raw(src, n)); + ASSERT_TRUE(verify_equal(cc, src, n)); + ASSERT_EQ(dto_batch_crc(b, 4), ref_crc_raw(src, n)); + ASSERT_EQ(dto_batch_result(b, 5), 1); + ASSERT_EQ(dto_batch_bytes_completed(b, 5), 777); + ASSERT_EQ(dto_batch_result(b, 6), 0); + } + dto_batch_destroy(b); + free(src); + free(src2); + free(mv); + free(fill); + free(d1); + free(d2); + free(cc); + free(pat); + return 0; +} + +static int test_batch_all_ops(void) +{ + return run_batch_all_ops(0); +} + +static int test_batch_single_op(void) +{ + return run_batch_all_ops(1); +} + +/* Operations the device cannot finish are redone by dto_batch_poll. Fresh + * anonymous pages are not mapped until first touched, so without + * block-on-fault the device stops at them; the results must still be exactly + * what the CPU computes, including for overlapping moves the device left + * half done in either direction. */ +static int test_batch_repair(void) +{ + const size_t page = 4096, region = 16 * page; + const uint64_t pattern = 0xA5A55A5A0F0FF0F0ULL; + uint8_t *m = mmap(NULL, 8 * region, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + uint8_t *up, *down, *cold, *fill, *cmp, *cc; + uint8_t *exp_up, *exp_down; + const size_t mv = 8 * page; /* overlapping move length */ + dto_batch *b = dto_batch_create(8); + int rc; + + ASSERT_TRUE(m != MAP_FAILED && b); + up = m; /* ascending move: low pages touched */ + down = m + region; /* descending move: high pages touched */ + cold = m + 2 * region; /* never touched: reads as zero */ + fill = m + 3 * region; + cmp = m + 4 * region; + cc = m + 5 * region; + exp_up = malloc(region); + exp_down = malloc(region); + ASSERT_TRUE(exp_up && exp_down); + + /* up: move [4p, 12p) down to [0, 8p); the device copies upward and + * stops at the untouched pages from 10p, after overwriting [4p, 6p) + * of its own source, so redoing the whole move would be wrong. */ + for (size_t i = 0; i < 10 * page; i++) + ((volatile uint8_t *)up)[i] = (uint8_t)(i * 7); + for (size_t i = 0; i < region; i++) + exp_up[i] = i < 10 * page ? (uint8_t)(i * 7) : 0; + for (size_t i = 0; i < mv; i++) /* forward copy is memmove here */ + exp_up[i] = exp_up[4 * page + i]; + + /* down: move [0, 8p) up to [4p, 12p); the device copies downward and + * stops at the untouched pages below 2p, after overwriting [6p, 8p) + * of its own source. */ + for (size_t i = 2 * page; i < region; i++) + ((volatile uint8_t *)down)[i] = (uint8_t)(i * 13); + for (size_t i = 0; i < region; i++) + exp_down[i] = i >= 2 * page ? (uint8_t)(i * 13) : 0; + for (size_t i = mv; i-- > 0;) /* backward copy is memmove here */ + exp_down[4 * page + i] = exp_down[i]; + + ASSERT_TRUE(dto_batch_add_memmove(b, up, up + 4 * page, mv, 0) == 0); + ASSERT_TRUE(dto_batch_add_memmove(b, down + 4 * page, down, mv, 0) == 1); + ASSERT_TRUE(dto_batch_add_memfill(b, fill, pattern, region, 0) == 2); + ASSERT_TRUE(dto_batch_add_crc(b, cold, region, 0) == 3); + ASSERT_TRUE(dto_batch_add_compval(b, cold, 0, region, 0) == 4); + ASSERT_TRUE(dto_batch_add_compare(b, cold, cmp, region, 0) == 5); + ASSERT_TRUE(dto_batch_add_memcpy_crc(b, cc, cold, region, 0) == 6); + + rc = dto_batch_submit(b); + CHECK_SUBMIT(rc); + if (rc == DTO_ASYNC_SUBMITTED) { + ASSERT_EQ(ASYNC_WAIT(dto_batch_poll(b)), DTO_ASYNC_DONE); + ASSERT_TRUE(verify_equal(up, exp_up, region)); + ASSERT_TRUE(verify_equal(down, exp_down, region)); + for (size_t i = 0; i < region; i++) + ASSERT_EQ(fill[i], (uint8_t)(pattern >> (8 * (i & 7)))); + ASSERT_EQ(dto_batch_crc(b, 3), ref_crc_raw(cold, region)); + ASSERT_EQ(dto_batch_result(b, 4), 0); + ASSERT_EQ(dto_batch_result(b, 5), 0); + ASSERT_EQ(dto_batch_crc(b, 6), ref_crc_raw(cold, region)); + ASSERT_TRUE(verify_set(cc, 0, region)); + } + dto_batch_destroy(b); + munmap(m, 8 * region); + free(exp_up); + free(exp_down); + return 0; +} + +/* Two threads polling one batch that needs repair: the overlapping move is + * not idempotent, so a repair done twice would show. */ +struct poll_arg { + dto_batch *b; + int rc; +}; + +static void *poller(void *p) +{ + struct poll_arg *a = p; + + a->rc = ASYNC_WAIT(dto_batch_poll(a->b)); + return NULL; +} + +static int test_batch_concurrent_poll(void) +{ + const size_t page = 4096, region = 16 * page, mv = 8 * page; + uint8_t *m = mmap(NULL, region, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + uint8_t *exp = malloc(region); + dto_batch *b = dto_batch_create(2); + struct poll_arg args[2]; + pthread_t t[2]; + int rc; + + ASSERT_TRUE(m != MAP_FAILED && exp && b); + for (size_t i = 2 * page; i < region; i++) + ((volatile uint8_t *)m)[i] = (uint8_t)(i * 3); + for (size_t i = 0; i < region; i++) + exp[i] = i >= 2 * page ? (uint8_t)(i * 3) : 0; + for (size_t i = mv; i-- > 0;) + exp[4 * page + i] = exp[i]; + + ASSERT_EQ(dto_batch_add_memmove(b, m + 4 * page, m, mv, 0), 0); + rc = dto_batch_submit(b); + CHECK_SUBMIT(rc); + if (rc == DTO_ASYNC_SUBMITTED) { + for (int i = 0; i < 2; i++) { + args[i].b = b; + pthread_create(&t[i], NULL, poller, &args[i]); + } + for (int i = 0; i < 2; i++) { + pthread_join(t[i], NULL); + ASSERT_EQ(args[i].rc, DTO_ASYNC_DONE); + } + ASSERT_TRUE(verify_equal(m, exp, region)); + } + dto_batch_destroy(b); + munmap(m, region); + free(exp); return 0; } @@ -696,9 +1262,10 @@ static int test_async_batch_copy(void) uint8_t *src[COUNT], *dst[COUNT]; void *srcv[COUNT], *dstv[COUNT]; size_t sizes[COUNT]; - dto_batch_op *bop; + dto_batch *b = dto_batch_create(COUNT); int rc; + ASSERT_TRUE(b != NULL); for (int i = 0; i < COUNT; i++) { sizes[i] = base * (i + 1) + i * 64; /* varied, some unaligned */ src[i] = malloc(sizes[i]); @@ -710,16 +1277,15 @@ static int test_async_batch_copy(void) dstv[i] = dst[i]; } - bop = dto_batch_op_new(); - ASSERT_TRUE(bop != NULL); - rc = dto_submit_batch_copy(bop, dstv, srcv, sizes, COUNT, DTO_SUBMIT_BOF); + /* more copies than the batch holds cannot be submitted */ + ASSERT_EQ(dto_submit_batch_copy(b, dstv, srcv, sizes, COUNT + 1, 0), + DTO_ASYNC_FALLBACK); + rc = dto_submit_batch_copy(b, dstv, srcv, sizes, COUNT, DTO_SUBMIT_CC); + CHECK_SUBMIT(rc); if (rc == DTO_ASYNC_SUBMITTED) { - rc = ASYNC_WAIT(dto_batch_poll(bop)); - ASSERT_EQ(rc, DTO_ASYNC_DONE); - /* a completed batch stays DONE */ - ASSERT_EQ(dto_batch_poll(bop), DTO_ASYNC_DONE); + dto_batch_wait(b); + ASSERT_EQ(ASYNC_WAIT(dto_batch_poll(b)), DTO_ASYNC_DONE); } else { - ASSERT_EQ(rc, DTO_ASYNC_FALLBACK); for (int i = 0; i < COUNT; i++) memcpy(dst[i], src[i], sizes[i]); } @@ -728,13 +1294,16 @@ static int test_async_batch_copy(void) free(src[i]); free(dst[i]); } - dto_batch_op_free(bop); + dto_batch_destroy(b); return 0; } int main(void) { - printf("DTO Functional Tests\n"); + expect_dsa = getenv("DTO_TEST_EXPECT_DSA") != NULL; + + printf("DTO Functional Tests%s\n", + expect_dsa ? " (DTO_TEST_EXPECT_DSA: offload required)" : ""); printf("====================\n\n"); struct dto_test tests[] = { @@ -749,11 +1318,23 @@ int main(void) TEST_ENTRY(test_memcpy_unaligned), TEST_ENTRY(test_multithread), TEST_ENTRY(test_async_memcpy), + TEST_ENTRY(test_async_memcpy_no_cc), TEST_ENTRY(test_async_memset), + TEST_ENTRY(test_async_memfill), TEST_ENTRY(test_async_crc), TEST_ENTRY(test_async_memcpy_crc), - TEST_ENTRY(test_async_memcpy_no_bof), + TEST_ENTRY(test_async_crc32c), + TEST_ENTRY(test_async_compare), + TEST_ENTRY(test_async_compval), + TEST_ENTRY(test_async_transl_fetch), + TEST_ENTRY(test_async_bof), + TEST_ENTRY(test_async_wait_unsubmitted), TEST_ENTRY(test_async_bad_args), + TEST_ENTRY(test_batch_limits), + TEST_ENTRY(test_batch_all_ops), + TEST_ENTRY(test_batch_single_op), + TEST_ENTRY(test_batch_repair), + TEST_ENTRY(test_batch_concurrent_poll), TEST_ENTRY(test_async_batch_copy), {NULL, NULL} }; From de1f694873d2badcd69810cbe18a88249b2503fa Mon Sep 17 00:00:00 2001 From: Daniel Byrne Date: Wed, 21 Oct 2026 08:26:46 -0700 Subject: [PATCH 5/5] docs: describe the explicit API, libdto_explicit and DTO_TEST_EXPECT_DSA Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Mv7xKtQPkGoXQUvvuuhAEz --- README.md | 47 +++++++++++++++++++++++++++++++++++++++++++++++ TESTING.md | 20 ++++++++++++++++++++ 2 files changed, 67 insertions(+) diff --git a/README.md b/README.md index bd3e239..42cda0e 100644 --- a/README.md +++ b/README.md @@ -71,6 +71,7 @@ Following environment variables control the behavior of DTO library: DTO_UMWAIT_DELAY=xxxx defines delay for umwait command (check max possible value at: /sys/devices/system/cpu/umwait_control/max_time), default is 100000 DTO_LOG_FILE= Redirect the DTO output to the specified file instead of std output (useful for debugging and statistics collection). file name is suffixed by process pid. DTO_LOG_LEVEL=0/1/2 controls the log level. higher value means more verbose logging (default 0). + DTO_CRC_MIN_BYTES=xxxx smallest buffer dto_crc32c_with_seed() sends to DSA; smaller ones are checksummed on the CPU (default: DTO_MIN_BYTES). ``` Although not the only usage models of DTO, the following are some common ones: @@ -95,6 +96,51 @@ Although not the only usage models of DTO, the following are some common ones: DTO_WAIT_METHOD=yield or umwait (saves either cycles or power) +## Explicit asynchronous API + +Besides intercepting the libc calls, libdto exports an explicit submit/poll +API, declared in `dto.h`, for applications that want to overlap DSA work with +their own: copy, fill, CRC32C (optionally fused with a copy), compare, compare +against a pattern and translation fetch, each as a single operation or batched +(up to 1024 operations in one DSA batch descriptor). + +```c +#include + +dto_async_op op; /* caller-owned, 64-byte aligned */ + +if (dto_submit_memcpy_crc(&op, dst, src, n, DTO_SUBMIT_CC) == + DTO_ASYNC_SUBMITTED) { + do_other_work(); + dto_async_wait(&op); /* waits with DTO_WAIT_METHOD */ + if (dto_async_poll(&op) == DTO_ASYNC_DONE) + crc = dto_async_crc_val(&op); + else + redo_on_cpu(); +} else { /* DTO_ASYNC_FALLBACK: nothing was done */ + redo_on_cpu(); +} +``` + +A batch is built with `dto_batch_create`, `dto_batch_add_*` and +`dto_batch_submit`; `dto_batch_poll` redoes on the CPU anything the device +could not finish, so once it returns `DTO_ASYNC_DONE` every result is valid. + +Explicit submits are not subject to `DTO_MIN_BYTES`: the caller decides what to +offload. `DTO_SUBMIT_BOF` makes the device wait for page faults to be +resolved; it requires a work queue configured with block-on-fault +(`accel-config config-wq --block-on-fault=1`), and on one without it the +device rejects the operation. Without the flag a page fault stops the +operation partway, on any work queue. `dto_async_crc_val` and `dto_batch_crc` +return the raw CRC32C (the SSE4.2 `crc32` instruction applied from 0, no final +inversion); the seeded `*_crc32c` calls return standard CRC32C and chain across +buffers. See `dto.h` for the full contract. + +Programs that only use this API, and must not have their own large libc copies +sent to DSA, can link `libdto_explicit` instead: the same library without the +memcpy/memset/memmove/memcmp interposers (`-ldto_explicit`, or +`DTO::dto_explicit` from CMake). + ## Build Pre-requisite packages: @@ -124,6 +170,7 @@ The accel-config and libnuma dependencies are optional and can be disabled at co | `DTO_NUMA_SUPPORT` | `ON` | Use libnuma for NUMA-aware work queue selection. When disabled, libdto does not link against libnuma and the `DTO_IS_NUMA_AWARE` environment variable has no effect (work queues are used in round-robin order). | | `DTO_STATIC_ACCEL_CONFIG` | `OFF` | Statically link libaccel-config into libdto.so instead of depending on the shared library at runtime. Requires `libaccel-config.a` to be installed. | | `DTO_STATIC_NUMA` | `OFF` | Statically link libnuma into libdto.so instead of depending on the shared library at runtime. Requires a `libnuma.a` compiled with `-fPIC`. | +| `DTO_BUILD_EXPLICIT` | `ON` | Also build `libdto_explicit`, the explicit API (`dto.h`) without the libc interposers. | Example: build without either dependency: ```bash diff --git a/TESTING.md b/TESTING.md index a491d31..d0e1b37 100644 --- a/TESTING.md +++ b/TESTING.md @@ -59,6 +59,26 @@ hardware -- DTO falls back to CPU transparently. ctest -R functional ``` +They also cover the explicit asynchronous API (`dto.h`): every submit and +batch operation is checked against the CPU result, including the CPU repair of +batch operations the device could not finish. `functional_explicit` runs the +same tests against `libdto_explicit`. + +Because every explicit operation may legitimately fall back to the CPU, a +broken offload path would still pass. On a machine with a usable work queue, +set `DTO_TEST_EXPECT_DSA` to require that operations on faulted-in buffers are +submitted and completed by the device: + +```bash +sudo DTO_TEST_EXPECT_DSA=1 ./dto-test-functional +sudo DTO_TEST_EXPECT_DSA=1 ./dto-test-functional-explicit +``` + +`test_async_bof` checks the block-on-fault behavior against whichever setting +the work queue has (it reports which), so run the suite once with +`block_on_fault` enabled and once without to cover both. It assumes all work +queues DTO uses share the same setting. + ## Performance Tests The performance test (`perf`) detects regressions in libdto by