...

src/host/readback.rs

@@ -26,6 +26,7 @@ impl Host {
     net
   }
 }
+
 /// See [`Host::readback`].
 struct ReadbackState<'a> {
   host: &'a Host,

src/main.rs

@@ -353,7 +353,7 @@ fn reduce_exprs(host: &Host, exprs: &[Net], opts: &RuntimeOpts) {
   for expr in exprs {
     let mut net = DynNet::new(&heap, opts.lazy_mode);
     dispatch_dyn_net!(&mut net => {
-      host.encode_net(net, Trg::port(run::Port::new_var(net.root.addr())), expr);
+      host.encode_net(net, Trg::port(net.root.as_var()), expr);
       let start_time = Instant::now();
       if opts.single_core {
         net.normal();

src/run.rs

@@ -62,6 +62,7 @@ mod net;
 mod node;
 mod parallel;
 mod port;
+mod tag;
 mod wire;
 
 pub use addr::*;
@@ -72,6 +73,7 @@ pub use linker::*;
 pub use net::*;
 pub use node::*;
 pub use port::*;
+pub use tag::*;
 pub use wire::*;
 
 pub type Lab = u16;

src/run/addr.rs

@@ -37,17 +37,21 @@ impl Addr {
 
   const HALF_MASK: usize = 0b1000;
 
-  /// Given an address to one word of a two-word allocation, returns the address
-  /// of the first word of that allocation.
+  /// TODO
   #[inline(always)]
-  pub(super) fn left_half(&self) -> Self {
-    Addr(self.0 & !Addr::HALF_MASK)
+  pub(super) fn floor(&self, align: Align) -> Self {
+    Addr(self.0 & (usize::MAX << align.tag_bits()))
   }
 
-  /// Given an address to one word of a two-word allocation, returns the address
-  /// of the other word of that allocation.
+  /// TODO
   #[inline(always)]
-  pub fn other_half(&self) -> Self {
-    Addr(self.0 ^ Addr::HALF_MASK)
+  pub(super) fn other(&self, align: Align) -> Self {
+    Addr(self.0 ^ (1 << align.tag_bits()))
+  }
+
+  /// TODO
+  #[inline(always)]
+  pub(super) fn offset(&self, words: usize) -> Self {
+    Addr(self.0 + (words << 3))
   }
 }

src/run/allocator.rs

@@ -1,27 +1,17 @@
 use super::*;
 
-/// The memory behind a two-word allocation.
-///
-/// This must be aligned to 16 bytes so that the left word's address always ends
-/// with `0b0000` and the right word's address always ends with `0b1000`.
-#[repr(C)]
-#[repr(align(16))]
-#[derive(Default)]
-pub(super) struct Node(pub AtomicU64, pub AtomicU64);
-
 /// The memory buffer backing a [`Net`].
-#[repr(align(16))]
-pub struct Heap(pub(super) [Node]);
+#[repr(align(64))]
+pub struct Heap(pub(super) [AtomicU64]);
 
 impl Heap {
   #[inline]
   /// Allocate a new heap with a given size in words.
   pub fn new_words(words: usize) -> Box<Self> {
-    let nodes = words / 2;
     unsafe {
       Box::from_raw(core::ptr::slice_from_raw_parts_mut(
-        alloc::alloc(Layout::array::<Node>(nodes).unwrap()) as *mut _,
-        nodes,
+        alloc::alloc(Layout::array::<AtomicU64>(words).unwrap().align_to(64).unwrap()) as *mut _,
+        words,
       ) as *mut _)
     }
   }
@@ -37,66 +27,101 @@ pub struct Allocator<'h> {
   pub(super) tracer: Tracer,
   pub(super) heap: &'h Heap,
   pub(super) next: usize,
-  pub(super) head: Addr,
+  pub(super) heads: [Addr; 4],
 }
 
 deref!({<'h>} Allocator<'h> => self.tracer: Tracer);
 
+impl Align {
+  #[inline(always)]
+  const fn free(self) -> u64 {
+    (1 << self as u64) << 60
+  }
+}
+
+/// Sentinel values used to indicate free memory.
+impl Port {
+  pub(super) const FREE_1: Port = Port(Align1.free());
+  pub(super) const FREE_2: Port = Port(Align2.free());
+  pub(super) const FREE_4: Port = Port(Align4.free());
+  pub(super) const FREE_8: Port = Port(Align8.free());
+}
+
 impl<'h> Allocator<'h> {
   pub fn new(heap: &'h Heap) -> Self {
-    Allocator { tracer: Tracer::default(), heap, next: 0, head: Addr::NULL }
+    Allocator { tracer: Tracer::default(), heap, next: 0, heads: [Addr::NULL; 4] }
+  }
+
+  fn head(&mut self, align: Align) -> &mut Addr {
+    unsafe { self.heads.get_unchecked_mut(align as usize) }
   }
 
-  /// Frees one word of a two-word allocation.
+  fn push_addr(head: &mut Addr, addr: Addr) {
+    addr.val().store(head.0 as u64, Relaxed);
+    *head = addr;
+  }
+
+  /// Frees one word of an allocation of size `alloc_align`.
   #[inline(always)]
-  pub fn half_free(&mut self, addr: Addr) {
-    trace!(self.tracer, addr);
-    const FREE: u64 = Port::FREE.0;
+  pub fn free_word(&mut self, mut addr: Addr, alloc_align: Align) {
     if cfg!(feature = "_fuzz") {
       if cfg!(not(feature = "_fuzz_no_free")) {
-        assert_ne!(addr.val().swap(FREE, Relaxed), FREE, "double free");
+        let free = Port::FREE_1.0;
+        assert_ne!(addr.val().swap(free, Relaxed), free, "double free");
       }
-    } else {
-      addr.val().store(FREE, Relaxed);
-      if addr.other_half().val().load(Relaxed) == FREE {
-        trace!(self.tracer, "other free");
-        let addr = addr.left_half();
-        if addr.val().compare_exchange(FREE, self.head.0 as u64, Relaxed, Relaxed).is_ok() {
-          let old_head = &self.head;
-          let new_head = addr;
-          trace!(self.tracer, "appended", old_head, new_head);
-          self.head = new_head;
-        } else {
-          trace!(self.tracer, "too slow");
-        };
+      return;
+    }
+    let mut align = Align1;
+    if align == alloc_align {
+      return Self::push_addr(self.head(align), addr);
+    }
+    addr.val().store(align.free(), Relaxed);
+    while align != alloc_align {
+      if addr.other(align).val().load(Relaxed) != align.free() {
+        return;
+      }
+      trace!(self.tracer, "other free");
+      let next_align = unsafe { align.next().unwrap_unchecked() };
+      addr = addr.floor(next_align);
+      let next_value = if next_align == alloc_align { self.head(alloc_align).0 as u64 } else { next_align.free() };
+      if addr.val().compare_exchange(align.free(), next_value, Relaxed, Relaxed).is_err() {
+        return trace!(self.tracer, "too slow");
       }
+      trace!(self.tracer, "success");
+      if next_align == alloc_align {
+        let old_head = next_value;
+        let new_head = addr;
+        trace!(self.tracer, "appended", old_head, new_head);
+        return *self.head(align) = addr;
+      }
+      align = next_align;
     }
   }
 
-  /// Allocates a two-word node.
+  /// Allocates a node, with a size specified by `align`.
   #[inline(never)]
-  pub fn alloc(&mut self) -> Addr {
-    trace!(self.tracer, self.head);
-    let addr = if self.head != Addr::NULL {
-      let addr = self.head.clone();
-      let next = Addr(self.head.val().load(Relaxed) as usize);
-      trace!(self.tracer, next);
-      self.head = next;
+  pub fn alloc(&mut self, align: Align) -> Addr {
+    let head = self.head(align);
+    let addr = if *head != Addr::NULL {
+      let addr = *head;
+      let next = Addr(head.val().load(Relaxed) as usize);
+      *head = next;
       addr
     } else {
       let index = self.next;
-      self.next += 1;
-      Addr(&self.heap.0.get(index).expect("OOM").0 as *const _ as _)
+      self.next += 8;
+      Addr(&self.heap.0.get(index).expect("OOM") as *const _ as _)
     };
-    trace!(self.tracer, addr, self.head);
-    addr.val().store(Port::LOCK.0, Relaxed);
-    addr.other_half().val().store(Port::LOCK.0, Relaxed);
+    trace!(self, addr);
+    for i in 0 .. align.width() {
+      addr.offset(i as usize).val().store(Port::LOCK.0, Relaxed);
+    }
     addr
   }
 
   #[inline(always)]
   pub(crate) fn free_wire(&mut self, wire: Wire) {
-    self.half_free(wire.addr());
+    self.free_word(wire.addr(), wire.alloc_align());
   }
 
   /// If `trg` is a wire, frees the backing memory.

src/run/def.rs

@@ -186,7 +186,7 @@ impl<'a, M: Mode> Net<'a, M> {
 
     let def = port.addr().def();
 
-    if trg.tag() == Ctr && !def.labs.has(trg.lab()) {
+    if trg.is_ctr_ish() && !def.labs.has(trg.lab()) {
       return self.comm02(port, trg);
     }
 

src/run/instruction.rs

@@ -112,14 +112,14 @@ impl<'a, M: Mode> Net<'a, M> {
   #[inline(always)]
   pub(crate) fn do_ctr(&mut self, lab: Lab, trg: Trg) -> (Trg, Trg) {
     let port = trg.target();
-    if !M::LAZY && port.tag() == Ctr && port.lab() == lab {
-      trace!(self.tracer, "fast");
+    if !M::LAZY && port.is(Tag::Ctr) && port.lab() == lab {
+      trace!(self, "fast");
       self.free_trg(trg);
       let node = port.consume_node();
       self.rwts.anni += 1;
       (Trg::wire(node.p1), Trg::wire(node.p2))
     // TODO: fast copy?
-    } else if false && !M::LAZY && port.tag() == Num || port.tag() == Ref && lab >= port.lab() {
+    } else if false && !M::LAZY && port.is(Tag::Num) || port.is(Tag::Ref) && lab >= port.lab() {
       self.rwts.comm += 1;
       (Trg::port(port.clone()), Trg::port(port))
     } else {
@@ -170,7 +170,7 @@ impl<'a, M: Mode> Net<'a, M> {
   #[inline(always)]
   pub(crate) fn do_mat(&mut self, trg: Trg) -> (Trg, Trg) {
     let port = trg.target();
-    if !M::LAZY && port.tag() == Num {
+    if !M::LAZY && port.is(Tag::Num) {
       self.rwts.oper += 1;
       self.free_trg(trg);
       let num = port.num();
@@ -198,13 +198,13 @@ impl<'a, M: Mode> Net<'a, M> {
 
   #[inline(always)]
   pub(crate) fn do_wires(&mut self) -> (Trg, Trg, Trg, Trg) {
-    let a = self.alloc();
-    let b = a.other_half();
+    let a = self.alloc(Align2);
+    let b = a.offset(1);
     (
-      Trg::port(Port::new_var(a.clone())),
-      Trg::wire(Wire::new(a)),
-      Trg::port(Port::new_var(b.clone())),
-      Trg::wire(Wire::new(b)),
+      Trg::port(Port::new_var(Align2, a.clone())),
+      Trg::wire(Wire::new(Align2, a)),
+      Trg::port(Port::new_var(Align2, b.clone())),
+      Trg::wire(Wire::new(Align2, b)),
     )
   }
 
@@ -213,7 +213,7 @@ impl<'a, M: Mode> Net<'a, M> {
   #[allow(unused)] // TODO: emit this instruction
   pub(crate) fn do_mat_con_con(&mut self, trg: Trg, out: Trg) -> (Trg, Trg, Trg) {
     let port = trg.target();
-    if !M::LAZY && trg.target().tag() == Num {
+    if !M::LAZY && trg.target().is(Tag::Num) {
       self.rwts.oper += 1;
       self.free_trg(trg);
       let num = port.num();
@@ -241,7 +241,7 @@ impl<'a, M: Mode> Net<'a, M> {
   #[allow(unused)] // TODO: emit this instruction
   pub(crate) fn do_mat_con(&mut self, trg: Trg, out: Trg) -> (Trg, Trg) {
     let port = trg.target();
-    if trg.target().tag() == Num {
+    if trg.target().is(Tag::Num) {
       self.rwts.oper += 1;
       self.free_trg(trg);
       let num = port.num();