Maintenance.

CHANGELOG.md

@@ -1,5 +1,19 @@
 The changelog explains changes pulled through from the private development repository. Bug fixes and small enhancements are committed between releases and not documented here.
 
+## 0.3.7 (August 14, 2023)
+
+- Path Oblivious Heap (@tskovlund)
+- Adjust batch and bucket size to program
+- Direct communication available in more protocols
+- Option for seed in fake preprocessing (@strieflin)
+- Lower memory usage due to improved register allocation
+- New instructions to speed up CISC compilation
+- Protocol implementation example
+- Fixed security bug: missing MAC checks in multi-threaded programs
+- Fixed security bug: race condition in MAC check
+- Fixed security bug: missing shuffling check in PS mod 2^k and Brain
+- Fixed security bug: insufficient drowning in pairwise protocols
+
 ## 0.3.6 (May 9, 2023)
 
 - More extensive benchmarking outputs

Compiler/GC/types.py

@@ -781,8 +781,6 @@ def store_in_mem(self, address):
  for i in range(n):
  for j, x in enumerate(v[i].bit_decompose()):
  x.store_in_mem(address + i + j * n)
- def reveal(self):
- return util.untuplify([x.reveal() for x in self.elements()])
  @classmethod
  def two_power(cls, nn, size=1):
  return cls.from_vec(
@@ -919,8 +917,7 @@ def bit_decompose(self, n_bits=None, security=None, maybe_mixed=None):
  return self.v[:n_bits]
  bit_compose = from_vec
  def reveal(self):
- assert len(self) == 1
- return self.v[0].reveal()
+ return util.untuplify([x.reveal() for x in self.elements()])
  def long_one(self):
  return [x.long_one() for x in self.v]
  def __rsub__(self, other):
@@ -1279,7 +1276,8 @@ def pow2(self, k):
 
 class sbitintvec(sbitvec, _bitint, _number, _sbitintbase):
  """
- Vector of signed integers for parallel binary computation::
+ Vector of signed integers for parallel binary computation.
+ The following example uses vectors of size two::
 
  sb32 = sbits.get_type(32)
  siv32 = sbitintvec.get_type(32)
@@ -1291,7 +1289,7 @@ class sbitintvec(sbitvec, _bitint, _number, _sbitintbase):
  print_ln('mul: %s, %s', c[0].reveal(), c[1].reveal())
  c = (a - b).elements()
  print_ln('sub: %s, %s', c[0].reveal(), c[1].reveal())
- c = (a < b).bit_decompose()
+ c = (a < b).elements()
  print_ln('lt: %s, %s', c[0].reveal(), c[1].reveal())
 
  This should output::
@@ -1467,7 +1465,7 @@ class sbitfixvec(_fix, _vec):
  print_ln('mul: %s, %s', c[0].reveal(), c[1].reveal())
  c = (a - b).elements()
  print_ln('sub: %s, %s', c[0].reveal(), c[1].reveal())
- c = (a < b).bit_decompose()
+ c = (a < b).elements()
  print_ln('lt: %s, %s', c[0].reveal(), c[1].reveal())
 
  This should output roughly::

Compiler/allocator.py

@@ -43,7 +43,7 @@ def pop(self, size):
  else:
  done = False
  for x in self.by_logsize[logsize + 1:]:
- for block_size, addresses in x.items():
+ for block_size, addresses in sorted(x.items()):
  if len(addresses) > 0:
  done = True
  break
@@ -60,16 +60,92 @@ def pop(self, size):
  self.by_address[addr + size] = diff
  return addr
 
+class AllocRange:
+ def __init__(self, base=0):
+ self.base = base
+ self.top = base
+ self.limit = base
+ self.grow = True
+ self.pool = defaultdict(set)
+
+ def alloc(self, size):
+ if self.pool[size]:
+ return self.pool[size].pop()
+ elif self.grow or self.top + size <= self.limit:
+ res = self.top
+ self.top += size
+ self.limit = max(self.limit, self.top)
+ if res >= REG_MAX:
+ raise RegisterOverflowError()
+ return res
+
+ def free(self, base, size):
+ assert self.base <= base < self.top
+ self.pool[size].add(base)
+
+ def stop_growing(self):
+ self.grow = False
+
+ def consolidate(self):
+ regs = []
+ for size, pool in self.pool.items():
+ for base in pool:
+ regs.append((base, size))
+ for base, size in reversed(sorted(regs)):
+ if base + size == self.top:
+ self.top -= size
+ self.pool[size].remove(base)
+ regs.pop()
+ else:
+ if program.Program.prog.verbose:
+ print('cannot free %d register blocks '
+ 'by a gap of %d at %d' %
+ (len(regs), self.top - size - base, base))
+ break
+
+class AllocPool:
+ def __init__(self):
+ self.ranges = defaultdict(lambda: [AllocRange()])
+ self.by_base = {}
+
+ def alloc(self, reg_type, size):
+ for r in self.ranges[reg_type]:
+ res = r.alloc(size)
+ if res is not None:
+ self.by_base[reg_type, res] = r
+ return res
+
+ def free(self, reg):
+ r = self.by_base.pop((reg.reg_type, reg.i))
+ r.free(reg.i, reg.size)
+
+ def new_ranges(self, min_usage):
+ for t, n in min_usage.items():
+ r = self.ranges[t][-1]
+ assert (n >= r.limit)
+ if r.limit < n:
+ r.stop_growing()
+ self.ranges[t].append(AllocRange(n))
+
+ def consolidate(self):
+ for r in self.ranges.values():
+ for rr in r:
+ rr.consolidate()
+
+ def n_fragments(self):
+ return max(len(r) for r in self.ranges)
+
 class StraightlineAllocator:
  """Allocate variables in a straightline program using n registers.
  It is based on the precondition that every register is only defined once."""
  def __init__(self, n, program):
  self.alloc = dict_by_id()
- self.usage = Compiler.program.RegType.create_dict(lambda: 0)
+ self.max_usage = defaultdict(lambda: 0)
  self.defined = dict_by_id()
  self.dealloc = set_by_id()
- self.n = n
+ assert(n == REG_MAX)
  self.program = program
+ self.old_pool = None
 
  def alloc_reg(self, reg, free):
  base = reg.vectorbase
@@ -79,14 +155,7 @@ def alloc_reg(self, reg, free):
 
  reg_type = reg.reg_type
  size = base.size
- if free[reg_type, size]:
- res = free[reg_type, size].pop()
- else:
- if self.usage[reg_type] < self.n:
- res = self.usage[reg_type]
- self.usage[reg_type] += size
- else:
- raise RegisterOverflowError()
+ res = free.alloc(reg_type, size)
  self.alloc[base] = res
 
  base.i = self.alloc[base]
@@ -126,7 +195,7 @@ def dealloc_reg(self, reg, inst, free):
  for x in itertools.chain(dup.duplicates, base.duplicates):
  to_check.add(x)
 
- free[reg.reg_type, base.size].append(self.alloc[base])
+ free.free(base)
  if inst.is_vec() and base.vector:
  self.defined[base] = inst
  for i in base.vector:
@@ -135,6 +204,7 @@ def dealloc_reg(self, reg, inst, free):
  self.defined[reg] = inst
 
  def process(self, program, alloc_pool):
+ self.update_usage(alloc_pool)
  for k,i in enumerate(reversed(program)):
  unused_regs = []
  for j in i.get_def():
@@ -161,12 +231,26 @@ def process(self, program, alloc_pool):
  if k % 1000000 == 0 and k > 0:
  print("Allocated registers for %d instructions at" % k, time.asctime())
 
+ self.update_max_usage(alloc_pool)
+ alloc_pool.consolidate()
+
  # print "Successfully allocated registers"
  # print "modp usage: %d clear, %d secret" % \
  # (self.usage[Compiler.program.RegType.ClearModp], self.usage[Compiler.program.RegType.SecretModp])
  # print "GF2N usage: %d clear, %d secret" % \
  # (self.usage[Compiler.program.RegType.ClearGF2N], self.usage[Compiler.program.RegType.SecretGF2N])
- return self.usage
+ return self.max_usage
+
+ def update_max_usage(self, alloc_pool):
+ for t, r in alloc_pool.ranges.items():
+ self.max_usage[t] = max(self.max_usage[t], r[-1].limit)
+
+ def update_usage(self, alloc_pool):
+ if self.old_pool:
+ self.update_max_usage(self.old_pool)
+ if id(self.old_pool) != id(alloc_pool):
+ alloc_pool.new_ranges(self.max_usage)
+ self.old_pool = alloc_pool
 
  def finalize(self, options):
  for reg in self.alloc:
@@ -178,6 +262,21 @@ def finalize(self, options):
  '\t\t'))
  if options.stop:
  sys.exit(1)
+ if self.program.verbose:
+ def p(sizes):
+ total = defaultdict(lambda: 0)
+ for (t, size) in sorted(sizes):
+ n = sizes[t, size]
+ total[t] += size * n
+ print('%s:%d*%d' % (t, size, n), end=' ')
+ print()
+ print('Total:', dict(total))
+
+ sizes = defaultdict(lambda: 0)
+ for reg in self.alloc:
+ x = reg.reg_type, reg.size
+ print('Used registers: ', end='')
+ p(sizes)
 
 def determine_scope(block, options):
  last_def = defaultdict_by_id(lambda: -1)

Compiler/circuit.py

@@ -10,7 +10,7 @@
 """
 
 from Compiler.GC.types import *
-from Compiler.library import function_block
+from Compiler.library import function_block, get_tape
 from Compiler import util
 import itertools
 import struct
@@ -54,7 +54,7 @@ def __call__(self, *inputs):
  return self.run(*inputs)
 
  def run(self, *inputs):
- n = inputs[0][0].n
+ n = inputs[0][0].n, get_tape()
  if n not in self.functions:
  self.functions[n] = function_block(lambda *args:
  self.compile(*args))

Compiler/compilerLib.py

@@ -270,9 +270,9 @@ def build_program(self, name=None):
  self.prog = Program(self.args, self.options, name=name)
  if self.execute:
  if self.options.execute in \
- ("emulate", "ring", "rep-field"):
+ ("emulate", "ring", "rep-field", "rep4-ring"):
  self.prog.use_trunc_pr = True
- if self.options.execute in ("ring",):
+ if self.options.execute in ("ring", "ps-rep-ring", "sy-rep-ring"):
  self.prog.use_split(3)
  if self.options.execute in ("semi2k",):
  self.prog.use_split(2)
@@ -487,6 +487,7 @@ def local_execution(self, args=[]):
  "Cannot produce %s. " % executable + \
  "Note that compilation requires a few GB of RAM.")
  vm = "%s/Scripts/%s.sh" % (self.root, self.options.execute)
+ sys.stdout.flush()
  os.execl(vm, vm, self.prog.name, *args)
 
  def remote_execution(self, args=[]):

Compiler/decision_tree.py

@@ -633,7 +633,8 @@ def preprocess_pandas(data):
  res.append(data.iloc[:,i].to_numpy())
  types.append('c')
  elif pandas.api.types.is_object_dtype(t):
- values = data.iloc[:,i].unique()
+ values = list(filter(lambda x: isinstance(x, str),
+ list(data.iloc[:,i].unique())))
  print('converting the following to unary:', values)
  if len(values) == 2:
  res.append(data.iloc[:,i].to_numpy() == values[1])

Compiler/instructions.py

@@ -638,6 +638,44 @@ class prefixsums(base.Instruction):
  code = base.opcodes['PREFIXSUMS']
  arg_format = ['sw','s']
 
+class picks(base.VectorInstruction):
+ """ Extract part of vector.
+
+ :param: result (sint)
+ :param: input (sint)
+ :param: start offset (int)
+ :param: step
+
+ """
+ __slots__ = []
+ code = base.opcodes['PICKS']
+ arg_format = ['sw','s','int','int']
+
+ def __init__(self, *args):
+ super(picks, self).__init__(*args)
+ assert 0 <= args[2] < len(args[1])
+ assert 0 <= args[2] + args[3] * len(args[0]) <= len(args[1])
+
+class concats(base.VectorInstruction):
+ """ Concatenate vectors.
+
+ :param: result (sint)
+ :param: start offset (int)
+ :param: input (sint)
+ :param: (repeat from offset)...
+
+ """
+ __slots__ = []
+ code = base.opcodes['CONCATS']
+ arg_format = tools.chain(['sw'], tools.cycle(['int','s']))
+
+ def __init__(self, *args):
+ super(concats, self).__init__(*args)
+ assert len(args) % 2 == 1
+ assert len(args[0]) == sum(args[1::2])
+ for i in range(1, len(args), 2):
+ assert args[i] == len(args[i + 1])
+
 @base.gf2n
 @base.vectorize
 class mulc(base.MulBase):