1 files changed, 153 insertions, 86 deletions

diff --git a/src/zix/ring.c b/src/zix/ring.c
index a48153a..d2d3195 100644
--- a/src/zix/ring.c
+++ b/src/zix/ring.c
@@ -1,12 +1,15 @@
-// Copyright 2011-2020 David Robillard <d@drobilla.net>
+// Copyright 2011-2022 David Robillard <d@drobilla.net>
 // SPDX-License-Identifier: ISC
 
 #include "zix/ring.h"
+#include "zix/common.h"
+
+#include "jalv_config.h"
 
 #include <stdlib.h>
 #include <string.h>
 
-#if HAVE_MLOCK
+#if USE_MLOCK
 #  include <sys/mman.h>
 #  define ZIX_MLOCK(ptr, size) mlock((ptr), (size))
 #elif defined(_WIN32)
@@ -17,124 +20,156 @@
 #  define ZIX_MLOCK(ptr, size)
 #endif
 
+/*
+  Note that for simplicity, only x86 and x64 are supported with MSVC.  Hopefully
+  stdatomic.h support arrives before anyone cares about running this code on
+  Windows on ARM.
+*/
 #if defined(_MSC_VER)
-#  include <windows.h>
-#  define ZIX_READ_BARRIER() MemoryBarrier()
-#  define ZIX_WRITE_BARRIER() MemoryBarrier()
-#elif defined(__GNUC__)
-#  define ZIX_READ_BARRIER() __atomic_thread_fence(__ATOMIC_ACQUIRE)
-#  define ZIX_WRITE_BARRIER() __atomic_thread_fence(__ATOMIC_RELEASE)
-#else
-#  pragma message("warning: No memory barriers, possible SMP bugs")
-#  define ZIX_READ_BARRIER()
-#  define ZIX_WRITE_BARRIER()
+#  include <intrin.h>
 #endif
 
 struct ZixRingImpl {
-  uint32_t write_head; ///< Read index into buf
-  uint32_t read_head;  ///< Write index into buf
-  uint32_t size;       ///< Size (capacity) in bytes
-  uint32_t size_mask;  ///< Mask for fast modulo
-  char*    buf;        ///< Contents
+  ZixAllocator* allocator;  ///< User allocator
+  uint32_t      write_head; ///< Read index into buf
+  uint32_t      read_head;  ///< Write index into buf
+  uint32_t      size;       ///< Size (capacity) in bytes
+  uint32_t      size_mask;  ///< Mask for fast modulo
+  char*         buf;        ///< Contents
 };
 
 static inline uint32_t
+zix_atomic_load(const uint32_t* const ptr)
+{
+#if defined(_MSC_VER)
+  const uint32_t val = *ptr;
+  _ReadBarrier();
+  return val;
+#else
+  return __atomic_load_n(ptr, __ATOMIC_ACQUIRE);
+#endif
+}
+
+static inline void
+zix_atomic_store(uint32_t* const ptr, // NOLINT(readability-non-const-parameter)
+                 const uint32_t  val)
+{
+#if defined(_MSC_VER)
+  _WriteBarrier();
+  *ptr = val;
+#else
+  __atomic_store_n(ptr, val, __ATOMIC_RELEASE);
+#endif
+}
+
+static inline uint32_t
 next_power_of_two(uint32_t size)
 {
   // http://graphics.stanford.edu/~seander/bithacks.html#RoundUpPowerOf2
   size--;
-  size |= size >> 1u;
-  size |= size >> 2u;
-  size |= size >> 4u;
-  size |= size >> 8u;
-  size |= size >> 16u;
+  size |= size >> 1U;
+  size |= size >> 2U;
+  size |= size >> 4U;
+  size |= size >> 8U;
+  size |= size >> 16U;
   size++;
   return size;
 }
 
 ZixRing*
-zix_ring_new(uint32_t size)
+zix_ring_new(ZixAllocator* const allocator, const uint32_t size)
 {
-  ZixRing* ring    = (ZixRing*)malloc(sizeof(ZixRing));
-  ring->write_head = 0;
-  ring->read_head  = 0;
-  ring->size       = next_power_of_two(size);
-  ring->size_mask  = ring->size - 1;
-  ring->buf        = (char*)malloc(ring->size);
+  ZixRing* ring = (ZixRing*)zix_malloc(allocator, sizeof(ZixRing));
+
+  if (ring) {
+    ring->allocator  = allocator;
+    ring->write_head = 0;
+    ring->read_head  = 0;
+    ring->size       = next_power_of_two(size);
+    ring->size_mask  = ring->size - 1;
+
+    if (!(ring->buf = (char*)zix_malloc(allocator, ring->size))) {
+      zix_free(allocator, ring);
+      return NULL;
+    }
+  }
+
   return ring;
 }
 
 void
-zix_ring_free(ZixRing* ring)
+zix_ring_free(ZixRing* const ring)
 {
   if (ring) {
-    free(ring->buf);
-    free(ring);
+    zix_free(ring->allocator, ring->buf);
+    zix_free(ring->allocator, ring);
   }
 }
 
 void
-zix_ring_mlock(ZixRing* ring)
+zix_ring_mlock(ZixRing* const ring)
 {
   ZIX_MLOCK(ring, sizeof(ZixRing));
   ZIX_MLOCK(ring->buf, ring->size);
 }
 
 void
-zix_ring_reset(ZixRing* ring)
+zix_ring_reset(ZixRing* const ring)
 {
   ring->write_head = 0;
   ring->read_head  = 0;
 }
 
+/*
+  General pattern for public thread-safe functions below: start with a single
+  atomic load of the "other's" index, then do whatever work, and finally end
+  with a single atomic store to "your" index (if it is changed).
+*/
+
 static inline uint32_t
-read_space_internal(const ZixRing* ring, uint32_t r, uint32_t w)
+read_space_internal(const ZixRing* const ring,
+                    const uint32_t       r,
+                    const uint32_t       w)
 {
-  if (r < w) {
-    return w - r;
-  }
-
-  return (w - r + ring->size) & ring->size_mask;
+  return (w - r) & ring->size_mask;
 }
 
 uint32_t
-zix_ring_read_space(const ZixRing* ring)
+zix_ring_read_space(const ZixRing* const ring)
 {
-  return read_space_internal(ring, ring->read_head, ring->write_head);
+  const uint32_t w = zix_atomic_load(&ring->write_head);
+
+  return read_space_internal(ring, ring->read_head, w);
 }
 
 static inline uint32_t
-write_space_internal(const ZixRing* ring, uint32_t r, uint32_t w)
+write_space_internal(const ZixRing* const ring,
+                     const uint32_t       r,
+                     const uint32_t       w)
 {
-  if (r == w) {
-    return ring->size - 1;
-  }
-
-  if (r < w) {
-    return ((r - w + ring->size) & ring->size_mask) - 1;
-  }
-
-  return (r - w) - 1;
+  return (r - w - 1U) & ring->size_mask;
 }
 
 uint32_t
-zix_ring_write_space(const ZixRing* ring)
+zix_ring_write_space(const ZixRing* const ring)
 {
-  return write_space_internal(ring, ring->read_head, ring->write_head);
+  const uint32_t r = zix_atomic_load(&ring->read_head);
+
+  return write_space_internal(ring, r, ring->write_head);
 }
 
 uint32_t
-zix_ring_capacity(const ZixRing* ring)
+zix_ring_capacity(const ZixRing* const ring)
 {
   return ring->size - 1;
 }
 
 static inline uint32_t
-peek_internal(const ZixRing* ring,
-              uint32_t       r,
-              uint32_t       w,
-              uint32_t       size,
-              void*          dst)
+peek_internal(const ZixRing* const ring,
+              const uint32_t       r,
+              const uint32_t       w,
+              const uint32_t       size,
+              void* const          dst)
 {
   if (read_space_internal(ring, r, w) < size) {
     return 0;
@@ -152,59 +187,91 @@ peek_internal(const ZixRing* ring,
 }
 
 uint32_t
-zix_ring_peek(ZixRing* ring, void* dst, uint32_t size)
+zix_ring_peek(ZixRing* const ring, void* const dst, const uint32_t size)
 {
-  return peek_internal(ring, ring->read_head, ring->write_head, size, dst);
+  const uint32_t w = zix_atomic_load(&ring->write_head);
+
+  return peek_internal(ring, ring->read_head, w, size, dst);
 }
 
 uint32_t
-zix_ring_read(ZixRing* ring, void* dst, uint32_t size)
+zix_ring_read(ZixRing* const ring, void* const dst, const uint32_t size)
 {
+  const uint32_t w = zix_atomic_load(&ring->write_head);
   const uint32_t r = ring->read_head;
-  const uint32_t w = ring->write_head;
-
-  if (peek_internal(ring, r, w, size, dst)) {
-    ZIX_READ_BARRIER();
-    ring->read_head = (r + size) & ring->size_mask;
-    return size;
+  if (!peek_internal(ring, r, w, size, dst)) {
+    return 0;
   }
 
-  return 0;
+  zix_atomic_store(&ring->read_head, (r + size) & ring->size_mask);
+  return size;
 }
 
 uint32_t
-zix_ring_skip(ZixRing* ring, uint32_t size)
+zix_ring_skip(ZixRing* const ring, const uint32_t size)
 {
+  const uint32_t w = zix_atomic_load(&ring->write_head);
   const uint32_t r = ring->read_head;
-  const uint32_t w = ring->write_head;
   if (read_space_internal(ring, r, w) < size) {
     return 0;
   }
 
-  ZIX_READ_BARRIER();
-  ring->read_head = (r + size) & ring->size_mask;
+  zix_atomic_store(&ring->read_head, (r + size) & ring->size_mask);
   return size;
 }
 
-uint32_t
-zix_ring_write(ZixRing* ring, const void* src, uint32_t size)
+ZixRingTransaction
+zix_ring_begin_write(ZixRing* const ring)
 {
-  const uint32_t r = ring->read_head;
+  const uint32_t r = zix_atomic_load(&ring->read_head);
   const uint32_t w = ring->write_head;
+
+  const ZixRingTransaction tx = {r, w};
+  return tx;
+}
+
+ZixStatus
+zix_ring_amend_write(ZixRing* const            ring,
+                     ZixRingTransaction* const tx,
+                     const void* const         src,
+                     const uint32_t            size)
+{
+  const uint32_t r = tx->read_head;
+  const uint32_t w = tx->write_head;
   if (write_space_internal(ring, r, w) < size) {
-    return 0;
+    return ZIX_STATUS_NO_MEM;
   }
 
-  if (w + size <= ring->size) {
+  const uint32_t end = w + size;
+  if (end <= ring->size) {
     memcpy(&ring->buf[w], src, size);
-    ZIX_WRITE_BARRIER();
-    ring->write_head = (w + size) & ring->size_mask;
+    tx->write_head = end & ring->size_mask;
   } else {
-    const uint32_t this_size = ring->size - w;
-    memcpy(&ring->buf[w], src, this_size);
-    memcpy(&ring->buf[0], (const char*)src + this_size, size - this_size);
-    ZIX_WRITE_BARRIER();
-    ring->write_head = size - this_size;
+    const uint32_t size1 = ring->size - w;
+    const uint32_t size2 = size - size1;
+    memcpy(&ring->buf[w], src, size1);
+    memcpy(&ring->buf[0], (const char*)src + size1, size2);
+    tx->write_head = size2;
+  }
+
+  return ZIX_STATUS_SUCCESS;
+}
+
+ZixStatus
+zix_ring_commit_write(ZixRing* const ring, const ZixRingTransaction* const tx)
+{
+  zix_atomic_store(&ring->write_head, tx->write_head);
+  return ZIX_STATUS_SUCCESS;
+}
+
+uint32_t
+zix_ring_write(ZixRing* const ring, const void* src, const uint32_t size)
+{
+  ZixRingTransaction tx = zix_ring_begin_write(ring);
+
+  if (zix_ring_amend_write(ring, &tx, src, size) ||
+      zix_ring_commit_write(ring, &tx)) {
+    return 0;
   }
 
   return size;