From 06b49a3419f35ee8c802f1f15b52cc45b546efb3 Mon Sep 17 00:00:00 2001 From: David Robillard Date: Wed, 10 Aug 2022 05:32:34 -0400 Subject: Update zix --- src/zix/ring.c | 239 ++++++++++++++++++++++++++++++++++++--------------------- 1 file changed, 153 insertions(+), 86 deletions(-) (limited to 'src/zix/ring.c') diff --git a/src/zix/ring.c b/src/zix/ring.c index a48153a..d2d3195 100644 --- a/src/zix/ring.c +++ b/src/zix/ring.c @@ -1,12 +1,15 @@ -// Copyright 2011-2020 David Robillard +// Copyright 2011-2022 David Robillard // SPDX-License-Identifier: ISC #include "zix/ring.h" +#include "zix/common.h" + +#include "jalv_config.h" #include #include -#if HAVE_MLOCK +#if USE_MLOCK # include # define ZIX_MLOCK(ptr, size) mlock((ptr), (size)) #elif defined(_WIN32) @@ -17,124 +20,156 @@ # define ZIX_MLOCK(ptr, size) #endif +/* + Note that for simplicity, only x86 and x64 are supported with MSVC. Hopefully + stdatomic.h support arrives before anyone cares about running this code on + Windows on ARM. +*/ #if defined(_MSC_VER) -# include -# define ZIX_READ_BARRIER() MemoryBarrier() -# define ZIX_WRITE_BARRIER() MemoryBarrier() -#elif defined(__GNUC__) -# define ZIX_READ_BARRIER() __atomic_thread_fence(__ATOMIC_ACQUIRE) -# define ZIX_WRITE_BARRIER() __atomic_thread_fence(__ATOMIC_RELEASE) -#else -# pragma message("warning: No memory barriers, possible SMP bugs") -# define ZIX_READ_BARRIER() -# define ZIX_WRITE_BARRIER() +# include #endif struct ZixRingImpl { - uint32_t write_head; ///< Read index into buf - uint32_t read_head; ///< Write index into buf - uint32_t size; ///< Size (capacity) in bytes - uint32_t size_mask; ///< Mask for fast modulo - char* buf; ///< Contents + ZixAllocator* allocator; ///< User allocator + uint32_t write_head; ///< Read index into buf + uint32_t read_head; ///< Write index into buf + uint32_t size; ///< Size (capacity) in bytes + uint32_t size_mask; ///< Mask for fast modulo + char* buf; ///< Contents }; +static inline uint32_t +zix_atomic_load(const uint32_t* const ptr) +{ +#if defined(_MSC_VER) + const uint32_t val = *ptr; + _ReadBarrier(); + return val; +#else + return __atomic_load_n(ptr, __ATOMIC_ACQUIRE); +#endif +} + +static inline void +zix_atomic_store(uint32_t* const ptr, // NOLINT(readability-non-const-parameter) + const uint32_t val) +{ +#if defined(_MSC_VER) + _WriteBarrier(); + *ptr = val; +#else + __atomic_store_n(ptr, val, __ATOMIC_RELEASE); +#endif +} + static inline uint32_t next_power_of_two(uint32_t size) { // http://graphics.stanford.edu/~seander/bithacks.html#RoundUpPowerOf2 size--; - size |= size >> 1u; - size |= size >> 2u; - size |= size >> 4u; - size |= size >> 8u; - size |= size >> 16u; + size |= size >> 1U; + size |= size >> 2U; + size |= size >> 4U; + size |= size >> 8U; + size |= size >> 16U; size++; return size; } ZixRing* -zix_ring_new(uint32_t size) +zix_ring_new(ZixAllocator* const allocator, const uint32_t size) { - ZixRing* ring = (ZixRing*)malloc(sizeof(ZixRing)); - ring->write_head = 0; - ring->read_head = 0; - ring->size = next_power_of_two(size); - ring->size_mask = ring->size - 1; - ring->buf = (char*)malloc(ring->size); + ZixRing* ring = (ZixRing*)zix_malloc(allocator, sizeof(ZixRing)); + + if (ring) { + ring->allocator = allocator; + ring->write_head = 0; + ring->read_head = 0; + ring->size = next_power_of_two(size); + ring->size_mask = ring->size - 1; + + if (!(ring->buf = (char*)zix_malloc(allocator, ring->size))) { + zix_free(allocator, ring); + return NULL; + } + } + return ring; } void -zix_ring_free(ZixRing* ring) +zix_ring_free(ZixRing* const ring) { if (ring) { - free(ring->buf); - free(ring); + zix_free(ring->allocator, ring->buf); + zix_free(ring->allocator, ring); } } void -zix_ring_mlock(ZixRing* ring) +zix_ring_mlock(ZixRing* const ring) { ZIX_MLOCK(ring, sizeof(ZixRing)); ZIX_MLOCK(ring->buf, ring->size); } void -zix_ring_reset(ZixRing* ring) +zix_ring_reset(ZixRing* const ring) { ring->write_head = 0; ring->read_head = 0; } +/* + General pattern for public thread-safe functions below: start with a single + atomic load of the "other's" index, then do whatever work, and finally end + with a single atomic store to "your" index (if it is changed). +*/ + static inline uint32_t -read_space_internal(const ZixRing* ring, uint32_t r, uint32_t w) +read_space_internal(const ZixRing* const ring, + const uint32_t r, + const uint32_t w) { - if (r < w) { - return w - r; - } - - return (w - r + ring->size) & ring->size_mask; + return (w - r) & ring->size_mask; } uint32_t -zix_ring_read_space(const ZixRing* ring) +zix_ring_read_space(const ZixRing* const ring) { - return read_space_internal(ring, ring->read_head, ring->write_head); + const uint32_t w = zix_atomic_load(&ring->write_head); + + return read_space_internal(ring, ring->read_head, w); } static inline uint32_t -write_space_internal(const ZixRing* ring, uint32_t r, uint32_t w) +write_space_internal(const ZixRing* const ring, + const uint32_t r, + const uint32_t w) { - if (r == w) { - return ring->size - 1; - } - - if (r < w) { - return ((r - w + ring->size) & ring->size_mask) - 1; - } - - return (r - w) - 1; + return (r - w - 1U) & ring->size_mask; } uint32_t -zix_ring_write_space(const ZixRing* ring) +zix_ring_write_space(const ZixRing* const ring) { - return write_space_internal(ring, ring->read_head, ring->write_head); + const uint32_t r = zix_atomic_load(&ring->read_head); + + return write_space_internal(ring, r, ring->write_head); } uint32_t -zix_ring_capacity(const ZixRing* ring) +zix_ring_capacity(const ZixRing* const ring) { return ring->size - 1; } static inline uint32_t -peek_internal(const ZixRing* ring, - uint32_t r, - uint32_t w, - uint32_t size, - void* dst) +peek_internal(const ZixRing* const ring, + const uint32_t r, + const uint32_t w, + const uint32_t size, + void* const dst) { if (read_space_internal(ring, r, w) < size) { return 0; @@ -152,59 +187,91 @@ peek_internal(const ZixRing* ring, } uint32_t -zix_ring_peek(ZixRing* ring, void* dst, uint32_t size) +zix_ring_peek(ZixRing* const ring, void* const dst, const uint32_t size) { - return peek_internal(ring, ring->read_head, ring->write_head, size, dst); + const uint32_t w = zix_atomic_load(&ring->write_head); + + return peek_internal(ring, ring->read_head, w, size, dst); } uint32_t -zix_ring_read(ZixRing* ring, void* dst, uint32_t size) +zix_ring_read(ZixRing* const ring, void* const dst, const uint32_t size) { + const uint32_t w = zix_atomic_load(&ring->write_head); const uint32_t r = ring->read_head; - const uint32_t w = ring->write_head; - - if (peek_internal(ring, r, w, size, dst)) { - ZIX_READ_BARRIER(); - ring->read_head = (r + size) & ring->size_mask; - return size; + if (!peek_internal(ring, r, w, size, dst)) { + return 0; } - return 0; + zix_atomic_store(&ring->read_head, (r + size) & ring->size_mask); + return size; } uint32_t -zix_ring_skip(ZixRing* ring, uint32_t size) +zix_ring_skip(ZixRing* const ring, const uint32_t size) { + const uint32_t w = zix_atomic_load(&ring->write_head); const uint32_t r = ring->read_head; - const uint32_t w = ring->write_head; if (read_space_internal(ring, r, w) < size) { return 0; } - ZIX_READ_BARRIER(); - ring->read_head = (r + size) & ring->size_mask; + zix_atomic_store(&ring->read_head, (r + size) & ring->size_mask); return size; } -uint32_t -zix_ring_write(ZixRing* ring, const void* src, uint32_t size) +ZixRingTransaction +zix_ring_begin_write(ZixRing* const ring) { - const uint32_t r = ring->read_head; + const uint32_t r = zix_atomic_load(&ring->read_head); const uint32_t w = ring->write_head; + + const ZixRingTransaction tx = {r, w}; + return tx; +} + +ZixStatus +zix_ring_amend_write(ZixRing* const ring, + ZixRingTransaction* const tx, + const void* const src, + const uint32_t size) +{ + const uint32_t r = tx->read_head; + const uint32_t w = tx->write_head; if (write_space_internal(ring, r, w) < size) { - return 0; + return ZIX_STATUS_NO_MEM; } - if (w + size <= ring->size) { + const uint32_t end = w + size; + if (end <= ring->size) { memcpy(&ring->buf[w], src, size); - ZIX_WRITE_BARRIER(); - ring->write_head = (w + size) & ring->size_mask; + tx->write_head = end & ring->size_mask; } else { - const uint32_t this_size = ring->size - w; - memcpy(&ring->buf[w], src, this_size); - memcpy(&ring->buf[0], (const char*)src + this_size, size - this_size); - ZIX_WRITE_BARRIER(); - ring->write_head = size - this_size; + const uint32_t size1 = ring->size - w; + const uint32_t size2 = size - size1; + memcpy(&ring->buf[w], src, size1); + memcpy(&ring->buf[0], (const char*)src + size1, size2); + tx->write_head = size2; + } + + return ZIX_STATUS_SUCCESS; +} + +ZixStatus +zix_ring_commit_write(ZixRing* const ring, const ZixRingTransaction* const tx) +{ + zix_atomic_store(&ring->write_head, tx->write_head); + return ZIX_STATUS_SUCCESS; +} + +uint32_t +zix_ring_write(ZixRing* const ring, const void* src, const uint32_t size) +{ + ZixRingTransaction tx = zix_ring_begin_write(ring); + + if (zix_ring_amend_write(ring, &tx, src, size) || + zix_ring_commit_write(ring, &tx)) { + return 0; } return size; -- cgit v1.2.1