From 5981f4deffaee0f1df6b681dbc45a8ed9de36af9 Mon Sep 17 00:00:00 2001 From: Joe Bryan Date: Thu, 22 Sep 2022 11:31:38 -0400 Subject: [PATCH 001/271] u3: rewrites jam atom encoding to maximize its range --- pkg/noun/serial.c | 54 +++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 47 insertions(+), 7 deletions(-) diff --git a/pkg/noun/serial.c b/pkg/noun/serial.c index 698a959d66..f994814fe1 100644 --- a/pkg/noun/serial.c +++ b/pkg/noun/serial.c @@ -36,7 +36,7 @@ struct _cs_jam_fib { /* _cs_jam_fib_grow(): reallocate buffer with fibonacci growth */ -static void +static inline void _cs_jam_fib_grow(struct _cs_jam_fib* fib_u, c3_w mor_w) { c3_w wan_w = fib_u->bit_w + mor_w; @@ -45,6 +45,7 @@ _cs_jam_fib_grow(struct _cs_jam_fib* fib_u, c3_w mor_w) // if ( wan_w < mor_w ) { u3m_bail(c3__fail); + return; } if ( wan_w > fib_u->a_w ) { @@ -64,7 +65,7 @@ _cs_jam_fib_grow(struct _cs_jam_fib* fib_u, c3_w mor_w) /* _cs_jam_fib_chop(): chop [met_w] bits of [a] into [fib_u] */ -static void +static inline void _cs_jam_fib_chop(struct _cs_jam_fib* fib_u, c3_w met_w, u3_noun a) { c3_w bit_w = fib_u->bit_w; @@ -86,12 +87,51 @@ _cs_jam_fib_mat(struct _cs_jam_fib* fib_u, u3_noun a) _cs_jam_fib_chop(fib_u, 1, 1); } else { - c3_w a_w = u3r_met(0, a); - c3_w b_w = c3_bits_word(a_w); + c3_w a_w = u3r_met(0, a); + c3_w b_w = c3_bits_word(a_w); + c3_w bit_w = fib_u->bit_w; + + // amortize overflow checks and reallocation + // + { + c3_w met_w = a_w + (2 * b_w); + + if ( a_w > (UINT32_MAX - 64) ) { + u3m_bail(c3__fail); + return; + } + + _cs_jam_fib_grow(fib_u, met_w); + fib_u->bit_w += met_w; + } - _cs_jam_fib_chop(fib_u, b_w+1, 1 << b_w); - _cs_jam_fib_chop(fib_u, b_w-1, a_w & ((1 << (b_w-1)) - 1)); - _cs_jam_fib_chop(fib_u, a_w, a); + { + c3_w src_w[2]; + c3_w* buf_w = fib_u->sab_u->buf_w; + + // _cs_jam_fib_chop(fib_u, b_w+1, 1 << b_w); + // + { + c3_d dat_d = (c3_d)1 << b_w; + src_w[0] = (c3_w)dat_d; + src_w[1] = dat_d >> 32; + + u3r_chop_words(0, 0, b_w + 1, bit_w, buf_w, 2, src_w); + bit_w += b_w + 1; + } + + // _cs_jam_fib_chop(fib_u, b_w-1, a_w); + // + { + src_w[0] = a_w; + u3r_chop_words(0, 0, b_w - 1, bit_w, buf_w, 1, src_w); + bit_w += b_w - 1; + } + + // _cs_jam_fib_chop(fib_u, a_w, a); + // + u3r_chop(0, 0, a_w, bit_w, buf_w, a); + } } } From b256c87f6ddf9ba42d983b39a9200e63027e5b7a Mon Sep 17 00:00:00 2001 From: Joe Bryan Date: Tue, 16 May 2023 23:26:10 -0400 Subject: [PATCH 002/271] jets: adds +hub --- pkg/noun/jets/c/hub.c | 61 +++++++++++++++++++++++++++++++++++++++++++ pkg/noun/jets/q.h | 1 + pkg/noun/jets/w.h | 1 + 3 files changed, 63 insertions(+) create mode 100644 pkg/noun/jets/c/hub.c diff --git a/pkg/noun/jets/c/hub.c b/pkg/noun/jets/c/hub.c new file mode 100644 index 0000000000..bae3831649 --- /dev/null +++ b/pkg/noun/jets/c/hub.c @@ -0,0 +1,61 @@ +#include "jets/q.h" +#include "jets/w.h" + +#include "noun.h" + +u3_noun +u3qc_hub(u3_atom a, u3_atom b) +{ + c3_w a_w, b_w, c_w; + + if ( 1 == a ) return u3k(b); + + a_w = u3r_met(0, a); + b_w = u3r_met(0, b); + + if ( b_w < a_w ) return u3m_bail(c3__exit); + + c_w = b_w - a_w; + + if ( (c3y == u3a_is_cat(a)) && (c3y == u3a_is_cat(b)) ) { + if ( a != (b >> c_w) ) return u3m_bail(c3__exit); + + return b & ((1 << c_w) - 1); + } + else { + u3_atom c, d, e, f, g; + + c = u3i_word(b_w - a_w); + d = u3qc_rsh(0, c, b); + + if ( c3n == u3r_sing(a, d) ) return u3m_bail(c3__exit); + + e = u3qc_bex(c); + f = u3qa_dec(e); + g = u3qc_dis(b, f); + + u3z(c); + u3z(d); + u3z(e); + u3z(f); + + return g; + } +} + +u3_noun +u3wc_hub(u3_noun cor) +{ + u3_noun a, b; + if ( (c3n == u3r_mean(cor, u3x_sam_2, &a, u3x_sam_3, &b, 0)) + || (0 == a) + || (0 == b) + || (c3n == u3ud(b)) + || (c3n == u3ud(a) && b != 1) ) + { + return u3m_bail(c3__exit); + } + else { + return u3qc_hub(a, b); + } +} diff --git a/pkg/noun/jets/q.h b/pkg/noun/jets/q.h index e9c8bfd9b3..ad4b10b82e 100644 --- a/pkg/noun/jets/q.h +++ b/pkg/noun/jets/q.h @@ -57,6 +57,7 @@ u3_noun u3qc_dvr(u3_atom, u3_atom); u3_noun u3qc_end(u3_atom, u3_atom, u3_atom); u3_noun u3qc_gor(u3_atom, u3_atom); + u3_noun u3qc_hub(u3_atom, u3_atom); u3_noun u3qc_lsh(u3_atom, u3_atom, u3_atom); u3_noun u3qc_mas(u3_atom); u3_noun u3qc_met(u3_atom, u3_atom); diff --git a/pkg/noun/jets/w.h b/pkg/noun/jets/w.h index 2898e0b6e6..76ff87d4be 100644 --- a/pkg/noun/jets/w.h +++ b/pkg/noun/jets/w.h @@ -58,6 +58,7 @@ u3_noun u3wc_dor(u3_noun); u3_noun u3wc_dvr(u3_noun); u3_noun u3wc_end(u3_noun); + u3_noun u3wc_hub(u3_noun); u3_noun u3wc_gor(u3_noun); u3_noun u3wc_lsh(u3_noun); u3_noun u3wc_mas(u3_noun); From af6adb8fb52cd94fa0e6061cc3611f913cf2759d Mon Sep 17 00:00:00 2001 From: Joe Bryan Date: Tue, 16 May 2023 23:29:00 -0400 Subject: [PATCH 003/271] jets: adds +pin --- pkg/noun/jets/c/pin.c | 54 +++++++++++++++++++++++++++++++++++++++++++ pkg/noun/jets/q.h | 1 + pkg/noun/jets/w.h | 1 + 3 files changed, 56 insertions(+) create mode 100644 pkg/noun/jets/c/pin.c diff --git a/pkg/noun/jets/c/pin.c b/pkg/noun/jets/c/pin.c new file mode 100644 index 0000000000..c1b45581f8 --- /dev/null +++ b/pkg/noun/jets/c/pin.c @@ -0,0 +1,54 @@ +#include "jets/q.h" +#include "jets/w.h" + +#include "noun.h" + +u3_noun +u3qc_pin(u3_atom a, u3_atom b) +{ + c3_w a_w, b_w, c_w; + + if ( 1 == a ) return c3y; + if ( 1 == b ) return c3n; + + a_w = u3r_met(0, a); + b_w = u3r_met(0, b); + + if ( b_w < a_w ) return c3n; + + c_w = b_w - a_w; + + if ( (c3y == u3a_is_cat(a)) && (c3y == u3a_is_cat(b)) ) { + return __(a == (b >> (b_w - a_w))); + } + else { + u3_atom c, d, e; + + c = u3i_word(b_w - a_w); + d = u3qc_rsh(0, c, b); + e = u3r_sing(a, d); + + u3z(c); + u3z(d); + + return e; + } +} + +u3_noun +u3wc_pin(u3_noun cor) +{ + u3_noun a, b; + + if ( (c3n == u3r_mean(cor, u3x_sam_2, &a, u3x_sam_3, &b, 0)) + || (0 == a) + || (0 == b) + || (c3n == u3ud(b)) + || (c3n == u3ud(a)) ) + { + return u3m_bail(c3__exit); + } + else { + return u3qc_pin(a, b); + } +} diff --git a/pkg/noun/jets/q.h b/pkg/noun/jets/q.h index ad4b10b82e..8a6cb15be7 100644 --- a/pkg/noun/jets/q.h +++ b/pkg/noun/jets/q.h @@ -65,6 +65,7 @@ u3_noun u3qc_mor(u3_atom, u3_atom); u3_noun u3qc_muk(u3_atom, u3_atom, u3_atom); u3_noun u3qc_peg(u3_atom, u3_atom); + u3_noun u3qc_pin(u3_atom, u3_atom); u3_noun u3qc_pow(u3_atom, u3_atom); u3_noun u3qc_rap(u3_atom, u3_noun); u3_noun u3qc_rep(u3_atom, u3_atom, u3_noun); diff --git a/pkg/noun/jets/w.h b/pkg/noun/jets/w.h index 76ff87d4be..df1d5c0fba 100644 --- a/pkg/noun/jets/w.h +++ b/pkg/noun/jets/w.h @@ -68,6 +68,7 @@ u3_noun u3wc_mug(u3_noun); u3_noun u3wc_muk(u3_noun); u3_noun u3wc_peg(u3_noun); + u3_noun u3wc_pin(u3_noun); u3_noun u3wc_pow(u3_noun); u3_noun u3wc_rap(u3_noun); u3_noun u3wc_rep(u3_noun); From 9d42e0dbb94a74b24b40ada50890a2938503940d Mon Sep 17 00:00:00 2001 From: Joe Bryan Date: Tue, 16 May 2023 23:32:13 -0400 Subject: [PATCH 004/271] jets: fixes unsafe c3_w -> atom conversion in +peg --- pkg/noun/jets/c/peg.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/pkg/noun/jets/c/peg.c b/pkg/noun/jets/c/peg.c index 28a8f3b2bd..16508c11af 100644 --- a/pkg/noun/jets/c/peg.c +++ b/pkg/noun/jets/c/peg.c @@ -14,16 +14,15 @@ return u3k(a); } - u3_atom c, d, e, f, g, h; + u3_atom d, e, f, g, h; + c3_w c_w = u3r_met(0, b) - 1; - c = u3r_met(0, b); - d = u3qa_dec(c); + d = u3i_word(c_w); e = u3qc_lsh(0, d, 1); f = u3qa_sub(b, e); g = u3qc_lsh(0, d, a); h = u3qa_add(f, g); - u3z(c); u3z(d); u3z(e); u3z(f); From b1d7d26cf738e73e8a05255af216817f002f0532 Mon Sep 17 00:00:00 2001 From: Joe Bryan Date: Tue, 16 May 2023 23:44:48 -0400 Subject: [PATCH 005/271] jets: optimizes direct +peg --- pkg/noun/jets/c/peg.c | 30 +++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/pkg/noun/jets/c/peg.c b/pkg/noun/jets/c/peg.c index 16508c11af..c87bff3c11 100644 --- a/pkg/noun/jets/c/peg.c +++ b/pkg/noun/jets/c/peg.c @@ -14,21 +14,29 @@ return u3k(a); } - u3_atom d, e, f, g, h; c3_w c_w = u3r_met(0, b) - 1; - d = u3i_word(c_w); - e = u3qc_lsh(0, d, 1); - f = u3qa_sub(b, e); - g = u3qc_lsh(0, d, a); - h = u3qa_add(f, g); + if ( (c3y == u3a_is_cat(a) && (c3y == u3a_is_cat(b))) ) { + c3_w d_w = b - (1 << c_w); + c3_d e_d = (c3_d)a << c_w; + return u3i_chub(d_w + e_d); + } + else { + u3_atom d, e, f, g, h; + + d = u3i_word(c_w); + e = u3qc_lsh(0, d, 1); + f = u3qa_sub(b, e); + g = u3qc_lsh(0, d, a); + h = u3qa_add(f, g); - u3z(d); - u3z(e); - u3z(f); - u3z(g); + u3z(d); + u3z(e); + u3z(f); + u3z(g); - return h; + return h; + } } u3_noun u3wc_peg(u3_noun cor) From 8fea31ec12657167fcb31a3201787deaeab1ca42 Mon Sep 17 00:00:00 2001 From: Joe Bryan Date: Tue, 16 May 2023 23:45:13 -0400 Subject: [PATCH 006/271] jets: restyles +peg --- pkg/noun/jets/c/peg.c | 93 +++++++++++++++++++++---------------------- 1 file changed, 45 insertions(+), 48 deletions(-) diff --git a/pkg/noun/jets/c/peg.c b/pkg/noun/jets/c/peg.c index c87bff3c11..cc39a1022b 100644 --- a/pkg/noun/jets/c/peg.c +++ b/pkg/noun/jets/c/peg.c @@ -1,57 +1,54 @@ -/// @file - #include "jets/q.h" #include "jets/w.h" #include "noun.h" +u3_noun +u3qc_peg(u3_atom a, u3_atom b) +{ + if ( 1 == b ) { + return u3k(a); + } - u3_noun - u3qc_peg(u3_atom a, - u3_atom b) - { - if ( 1 == b ) { - return u3k(a); - } - - c3_w c_w = u3r_met(0, b) - 1; - - if ( (c3y == u3a_is_cat(a) && (c3y == u3a_is_cat(b))) ) { - c3_w d_w = b - (1 << c_w); - c3_d e_d = (c3_d)a << c_w; - return u3i_chub(d_w + e_d); - } - else { - u3_atom d, e, f, g, h; - - d = u3i_word(c_w); - e = u3qc_lsh(0, d, 1); - f = u3qa_sub(b, e); - g = u3qc_lsh(0, d, a); - h = u3qa_add(f, g); - - u3z(d); - u3z(e); - u3z(f); - u3z(g); - - return h; - } + c3_w c_w = u3r_met(0, b) - 1; + + if ( (c3y == u3a_is_cat(a) && (c3y == u3a_is_cat(b))) ) { + c3_w d_w = b - (1 << c_w); + c3_d e_d = (c3_d)a << c_w; + return u3i_chub(d_w + e_d); + } + else { + u3_atom d, e, f, g, h; + + d = u3i_word(c_w); + e = u3qc_lsh(0, d, 1); + f = u3qa_sub(b, e); + g = u3qc_lsh(0, d, a); + h = u3qa_add(f, g); + + u3z(d); + u3z(e); + u3z(f); + u3z(g); + + return h; } - u3_noun - u3wc_peg(u3_noun cor) +} + +u3_noun +u3wc_peg(u3_noun cor) +{ + u3_noun a, b; + + if ( (c3n == u3r_mean(cor, u3x_sam_2, &a, u3x_sam_3, &b, 0)) || + (0 == a) || + (0 == b) || + (c3n == u3ud(b)) || + (c3n == u3ud(a) && b != 1) ) { - u3_noun a, b; - - if ( (c3n == u3r_mean(cor, u3x_sam_2, &a, u3x_sam_3, &b, 0)) || - (0 == a) || - (0 == b) || - (c3n == u3ud(b)) || - (c3n == u3ud(a) && b != 1) ) - { - return u3m_bail(c3__exit); - } else { - return u3qc_peg(a, b); - } + return u3m_bail(c3__exit); } - + else { + return u3qc_peg(a, b); + } +} From 30279b8990d2c87a7f1664e74b1d8c33a91af045 Mon Sep 17 00:00:00 2001 From: Joe Bryan Date: Tue, 16 May 2023 23:43:39 -0400 Subject: [PATCH 007/271] jets: fixes unsafe c3_w -> atom conversion in +mas --- pkg/noun/jets/c/mas.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/noun/jets/c/mas.c b/pkg/noun/jets/c/mas.c index 7b2658d738..881ae73977 100644 --- a/pkg/noun/jets/c/mas.c +++ b/pkg/noun/jets/c/mas.c @@ -17,8 +17,8 @@ return u3m_bail(c3__exit); } else { - c = u3qc_bex((b_w - 1)); - d = u3qc_bex((b_w - 2)); + c = u3qc_bex(u3i_word(b_w - 1)); + d = u3qc_bex(u3i_word(b_w - 2)); e = u3qa_sub(a, c); f = u3qc_con(e, d); From 76ce8392ba8032c2f7efc6ecb89d2ebea78d82d3 Mon Sep 17 00:00:00 2001 From: Joe Bryan Date: Tue, 16 May 2023 23:44:08 -0400 Subject: [PATCH 008/271] jets: optimizes direct +mas --- pkg/noun/jets/c/mas.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/noun/jets/c/mas.c b/pkg/noun/jets/c/mas.c index 881ae73977..63c8632079 100644 --- a/pkg/noun/jets/c/mas.c +++ b/pkg/noun/jets/c/mas.c @@ -16,6 +16,9 @@ if ( b_w < 2 ) { return u3m_bail(c3__exit); } + else if ( c3y == u3a_is_cat(a) ) { + return (a - (1 << (b_w - 1))) | (1 << (b_w - 2)); + } else { c = u3qc_bex(u3i_word(b_w - 1)); d = u3qc_bex(u3i_word(b_w - 2)); From 0b114c5c2c151daa6faa8c88a07806510ddf2252 Mon Sep 17 00:00:00 2001 From: Joe Bryan Date: Tue, 16 May 2023 23:45:48 -0400 Subject: [PATCH 009/271] jets: restyles +mas --- pkg/noun/jets/c/mas.c | 72 +++++++++++++++++++++---------------------- 1 file changed, 35 insertions(+), 37 deletions(-) diff --git a/pkg/noun/jets/c/mas.c b/pkg/noun/jets/c/mas.c index 63c8632079..4d5f0e45ca 100644 --- a/pkg/noun/jets/c/mas.c +++ b/pkg/noun/jets/c/mas.c @@ -1,48 +1,46 @@ -/// @file - #include "jets/q.h" #include "jets/w.h" #include "noun.h" +u3_noun +u3qc_mas(u3_atom a) +{ + c3_w b_w; + u3_atom c, d, e, f; - u3_noun - u3qc_mas(u3_atom a) - { - c3_w b_w; - u3_atom c, d, e, f; - - b_w = u3r_met(0, a); - if ( b_w < 2 ) { - return u3m_bail(c3__exit); - } - else if ( c3y == u3a_is_cat(a) ) { - return (a - (1 << (b_w - 1))) | (1 << (b_w - 2)); - } - else { - c = u3qc_bex(u3i_word(b_w - 1)); - d = u3qc_bex(u3i_word(b_w - 2)); - e = u3qa_sub(a, c); - f = u3qc_con(e, d); + b_w = u3r_met(0, a); + if ( b_w < 2 ) { + return u3m_bail(c3__exit); + } + else if ( c3y == u3a_is_cat(a) ) { + return (a - (1 << (b_w - 1))) | (1 << (b_w - 2)); + } + else { + c = u3qc_bex(u3i_word(b_w - 1)); + d = u3qc_bex(u3i_word(b_w - 2)); + e = u3qa_sub(a, c); + f = u3qc_con(e, d); - u3z(c); - u3z(d); - u3z(e); + u3z(c); + u3z(d); + u3z(e); - return f; - } + return f; } - u3_noun - u3wc_mas(u3_noun cor) - { - u3_noun a; +} - if ( (u3_none == (a = u3r_at(u3x_sam, cor))) || - (c3n == u3ud(a)) ) - { - return u3m_bail(c3__exit); - } else { - return u3qc_mas(a); - } - } +u3_noun +u3wc_mas(u3_noun cor) +{ + u3_noun a; + if ( (u3_none == (a = u3r_at(u3x_sam, cor))) || + (c3n == u3ud(a)) ) + { + return u3m_bail(c3__exit); + } + else { + return u3qc_mas(a); + } +} From 2f4c64ebc72d4d7dd8211a8e446b5508742d72ad Mon Sep 17 00:00:00 2001 From: Joe Bryan Date: Wed, 17 May 2023 10:58:13 -0400 Subject: [PATCH 010/271] jets: remove redundant expressions in +pin and +hub --- pkg/noun/jets/c/hub.c | 2 +- pkg/noun/jets/c/pin.c | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/noun/jets/c/hub.c b/pkg/noun/jets/c/hub.c index bae3831649..c4bf6e4726 100644 --- a/pkg/noun/jets/c/hub.c +++ b/pkg/noun/jets/c/hub.c @@ -25,7 +25,7 @@ u3qc_hub(u3_atom a, u3_atom b) else { u3_atom c, d, e, f, g; - c = u3i_word(b_w - a_w); + c = u3i_word(c_w); d = u3qc_rsh(0, c, b); if ( c3n == u3r_sing(a, d) ) return u3m_bail(c3__exit); diff --git a/pkg/noun/jets/c/pin.c b/pkg/noun/jets/c/pin.c index c1b45581f8..33d608f096 100644 --- a/pkg/noun/jets/c/pin.c +++ b/pkg/noun/jets/c/pin.c @@ -19,12 +19,12 @@ u3qc_pin(u3_atom a, u3_atom b) c_w = b_w - a_w; if ( (c3y == u3a_is_cat(a)) && (c3y == u3a_is_cat(b)) ) { - return __(a == (b >> (b_w - a_w))); + return __(a == (b >> c_w)); } else { u3_atom c, d, e; - c = u3i_word(b_w - a_w); + c = u3i_word(c_w); d = u3qc_rsh(0, c, b); e = u3r_sing(a, d); From e1adbfbae9cf6610f279dae680a3be9771dafa91 Mon Sep 17 00:00:00 2001 From: Joe Bryan Date: Wed, 17 May 2023 10:59:28 -0400 Subject: [PATCH 011/271] jets: corrects axis math in +hub --- pkg/noun/jets/c/hub.c | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/pkg/noun/jets/c/hub.c b/pkg/noun/jets/c/hub.c index c4bf6e4726..80214754d7 100644 --- a/pkg/noun/jets/c/hub.c +++ b/pkg/noun/jets/c/hub.c @@ -18,12 +18,16 @@ u3qc_hub(u3_atom a, u3_atom b) c_w = b_w - a_w; if ( (c3y == u3a_is_cat(a)) && (c3y == u3a_is_cat(b)) ) { + c3_w d_w; + if ( a != (b >> c_w) ) return u3m_bail(c3__exit); - return b & ((1 << c_w) - 1); + d_w = 1 << c_w; + + return (b & (d_w - 1)) | d_w; } else { - u3_atom c, d, e, f, g; + u3_atom c, d, e, f, g, h; c = u3i_word(c_w); d = u3qc_rsh(0, c, b); @@ -33,13 +37,15 @@ u3qc_hub(u3_atom a, u3_atom b) e = u3qc_bex(c); f = u3qa_dec(e); g = u3qc_dis(b, f); + h = u3qc_con(g, e); u3z(c); u3z(d); u3z(e); u3z(f); + u3z(g); - return g; + return h; } } From d57d1f69f4bca321d6365403f454a79a0bc20c26 Mon Sep 17 00:00:00 2001 From: Matthew LeVan Date: Wed, 12 Apr 2023 15:09:26 -0400 Subject: [PATCH 012/271] disk: add epoch handling to `u3_disk_init` --- pkg/vere/disk.c | 36 +++++++++++++++++++++++++++++++++--- pkg/vere/vere.h | 3 ++- 2 files changed, 35 insertions(+), 4 deletions(-) diff --git a/pkg/vere/disk.c b/pkg/vere/disk.c index 87c683fa9f..538238780e 100644 --- a/pkg/vere/disk.c +++ b/pkg/vere/disk.c @@ -867,7 +867,9 @@ u3_disk_init(c3_c* pax_c, u3_disk_cb cb_u) c3_free(dir_c); } - // create/load $pier/.urb/log, initialize db + // create/load $pier/.urb/log + // create/load $pier/.urb/log/0iN (epoch dir) + // initialize db // { c3_c* log_c = c3_malloc(10 + strlen(pax_c)); @@ -882,6 +884,30 @@ u3_disk_init(c3_c* pax_c, u3_disk_cb cb_u) return 0; } + // iterate through the log directory to find the latest epoch, + // or set it to "0i0" if none exist + // + u3_dent* den_u = log_u->com_u->all_u; + c3_c* lat_c; // latest epoch or "0i0" if none exist + if ( !den_u ) { + lat_c = "0i0"; + } else { + while ( den_u ) { + if ( 0 == strncmp(den_u->nam_c, "0i", 2) ) { + // update the lat_c if it's a later epoch + if ( !lat_c || (strcmp(den_u->nam_c, lat_c) > 0) ) { + lat_c = den_u->nam_c; + } + } + den_u = den_u->nex_u; + } + } + + // create/load $pier/.urb/log/0iN epoch directory + c3_c epo_c[8193]; + snprintf(epo_c, sizeof(epo_c), "%s/%s", log_c, lat_c); + c3_mkdir(epo_c, 0700); + // Arbitrarily choosing 1TB as a "large enough" mapsize // // per the LMDB docs: @@ -895,12 +921,16 @@ u3_disk_init(c3_c* pax_c, u3_disk_cb cb_u) #else 0x10000000000; #endif + + fprintf(stderr, "disk: initializing database at %s\r\n", epo_c); - if ( 0 == (log_u->mdb_u = u3_lmdb_init(log_c, siz_i)) ) { + if ( 0 == (log_u->mdb_u = u3_lmdb_init(epo_c, siz_i)) ) { fprintf(stderr, "disk: failed to initialize database\r\n"); - c3_free(log_c); c3_free(log_u); return 0; + } else { + // register the epoch directory in the disk struct + log_u->epo_u = u3_foil_folder(epo_c); } } diff --git a/pkg/vere/vere.h b/pkg/vere/vere.h index ca8a6da06d..ba06820a62 100644 --- a/pkg/vere/vere.h +++ b/pkg/vere/vere.h @@ -536,8 +536,9 @@ u3_dire* dir_u; // main pier directory u3_dire* urb_u; // urbit system data u3_dire* com_u; // log directory + u3_dire* epo_u; // current epoch directory c3_o liv_o; // live - void* mdb_u; // lmdb environment. + void* mdb_u; // lmdb environment c3_d sen_d; // commit requested c3_d dun_d; // committed u3_disk_cb cb_u; // callbacks From 4be7ec30102a409c7ac3a723e69653b97ce85009 Mon Sep 17 00:00:00 2001 From: Matthew LeVan Date: Thu, 13 Apr 2023 14:10:41 -0400 Subject: [PATCH 013/271] u3: add dir list to `u3_dire` --- pkg/vere/foil.c | 6 +++++- pkg/vere/vere.h | 3 ++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/pkg/vere/foil.c b/pkg/vere/foil.c index 26ff8b3d04..a241c32b66 100644 --- a/pkg/vere/foil.c +++ b/pkg/vere/foil.c @@ -106,13 +106,17 @@ u3_foil_folder(const c3_c* pax_c) dir_u = u3_dire_init(pax_c); } - /* create entries for all files + /* create entries for all files and directories */ while ( UV_EOF != uv_fs_scandir_next(&ruq_u, &den_u) ) { if ( UV_DIRENT_FILE == den_u.type ) { u3_dent* det_u = u3_dent_init(den_u.name); det_u->nex_u = dir_u->all_u; dir_u->all_u = det_u; + } else if ( UV_DIRENT_DIR == den_u.type ) { + u3_dent* det_u = u3_dent_init(den_u.name); + det_u->nex_u = dir_u->dir_u; + dir_u->dir_u = det_u; } } diff --git a/pkg/vere/vere.h b/pkg/vere/vere.h index ba06820a62..a1522c6e40 100644 --- a/pkg/vere/vere.h +++ b/pkg/vere/vere.h @@ -130,6 +130,7 @@ c3_c* pax_c; // path of directory uv_file fil_u; // file, opened read-only to fsync u3_dent* all_u; // file list + u3_dent* dir_u; // directory list } u3_dire; /* u3_save: checkpoint control. @@ -538,7 +539,7 @@ u3_dire* com_u; // log directory u3_dire* epo_u; // current epoch directory c3_o liv_o; // live - void* mdb_u; // lmdb environment + void* mdb_u; // lmdb env of current epoch c3_d sen_d; // commit requested c3_d dun_d; // committed u3_disk_cb cb_u; // callbacks From 89e919865a7422d7e558c09600edeeb79d34961f Mon Sep 17 00:00:00 2001 From: Matthew LeVan Date: Thu, 13 Apr 2023 15:02:20 -0400 Subject: [PATCH 014/271] disk: add `u3_disk_last_epoc` and `u3_disk_migrate` --- BUILD.bazel | 4 +- pkg/vere/disk.c | 113 +++++++++++++++++++++++++++++++++++------------- pkg/vere/main.c | 4 +- pkg/vere/vere.h | 11 +++++ 4 files changed, 97 insertions(+), 35 deletions(-) diff --git a/BUILD.bazel b/BUILD.bazel index 4ea5807432..304ecb66d4 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -44,8 +44,8 @@ config_setting( # Version flag for clang. string_flag( name = "clang_version", - # macOS uses `clang-14.0.0` by default. - build_setting_default = "14.0.0", + # macOS uses `clang-14.0.3` by default. + build_setting_default = "14.0.3", visibility = ["//visibility:public"], ) diff --git a/pkg/vere/disk.c b/pkg/vere/disk.c index 538238780e..867716ac74 100644 --- a/pkg/vere/disk.c +++ b/pkg/vere/disk.c @@ -2,6 +2,7 @@ #include "noun.h" #include "vere.h" +#include "version.h" #include "db/lmdb.h" struct _cd_read { @@ -22,6 +23,12 @@ struct _cd_save { struct _u3_disk* log_u; }; +/* DISK FORMAT + */ + +#define U3D_VER1 1 +#define U3D_VERLAT U3L_VER1 + #undef VERBOSE_DISK #undef DISK_TRACE_JAM #undef DISK_TRACE_CUE @@ -868,8 +875,7 @@ u3_disk_init(c3_c* pax_c, u3_disk_cb cb_u) } // create/load $pier/.urb/log - // create/load $pier/.urb/log/0iN (epoch dir) - // initialize db + // XX closures? // { c3_c* log_c = c3_malloc(10 + strlen(pax_c)); @@ -884,34 +890,26 @@ u3_disk_init(c3_c* pax_c, u3_disk_cb cb_u) return 0; } - // iterate through the log directory to find the latest epoch, - // or set it to "0i0" if none exist + // validation before migration // - u3_dent* den_u = log_u->com_u->all_u; - c3_c* lat_c; // latest epoch or "0i0" if none exist - if ( !den_u ) { - lat_c = "0i0"; - } else { - while ( den_u ) { - if ( 0 == strncmp(den_u->nam_c, "0i", 2) ) { - // update the lat_c if it's a later epoch - if ( !lat_c || (strcmp(den_u->nam_c, lat_c) > 0) ) { - lat_c = den_u->nam_c; - } - } - den_u = den_u->nex_u; - } - } - // create/load $pier/.urb/log/0iN epoch directory - c3_c epo_c[8193]; - snprintf(epo_c, sizeof(epo_c), "%s/%s", log_c, lat_c); - c3_mkdir(epo_c, 0700); + // migrate to the correct disk format + // + u3_disk_migrate(log_u); + + // find the latest epoch directory + // + c3_d lat_d; + u3_disk_last_epoc(log_u, &lat_d); + fprintf(stderr, "disk: latest epoch is 0i%" PRIu64 "\r\n", lat_d); - // Arbitrarily choosing 1TB as a "large enough" mapsize + // load latest epoch directory and set it in log_u // - // per the LMDB docs: - // "[..] on 64-bit there is no penalty for making this huge (say 1TB)." + c3_c epo_c[8193]; + snprintf(epo_c, 8192, "%s/0i%" PRIu64, log_c, lat_d); + log_u->epo_u = u3_foil_folder(epo_c); + + // initialize db of latest epoch // { const size_t siz_i = @@ -922,15 +920,10 @@ u3_disk_init(c3_c* pax_c, u3_disk_cb cb_u) 0x10000000000; #endif - fprintf(stderr, "disk: initializing database at %s\r\n", epo_c); - if ( 0 == (log_u->mdb_u = u3_lmdb_init(epo_c, siz_i)) ) { fprintf(stderr, "disk: failed to initialize database\r\n"); c3_free(log_u); return 0; - } else { - // register the epoch directory in the disk struct - log_u->epo_u = u3_foil_folder(epo_c); } } @@ -960,3 +953,61 @@ u3_disk_init(c3_c* pax_c, u3_disk_cb cb_u) return log_u; } + +/* u3_disk_last_epoc: get latest epoch number. + */ +c3_d u3_disk_last_epoc(u3_disk* log_u, c3_d* lat_d) { + c3_d ret_d = 1; // return code 1 if no epoch directories exist + *lat_d = 0; // initialize lat_d to 0 + u3_dent* den_u = log_u->com_u->dir_u; + while ( den_u ) { + c3_d epo_d = 0; + if ( 1 != sscanf(den_u->nam_c, "0i%" PRIu64, &epo_d) ) { + fprintf(stderr, "disk: epoch directory is not a @ui: %s\r\n", den_u->nam_c); + } else { + ret_d = 0; // return code 0 if at least one epoch directory exists + } + *lat_d = c3_max(epo_d, *lat_d); // update the latest epoch number + den_u = den_u->nex_u; + } + + return ret_d; +} + +/* u3_disk_migrate: migrates disk format. + */ +void u3_disk_migrate(u3_disk* log_u) +{ + c3_d lat_d; // latest epoch number + + // initialize disk v1 on fresh boots + // + if ( 1 == u3_disk_last_epoc(log_u, &lat_d) ) { // XX add another condition to ensure fresh boot + fprintf(stderr, "disk: migrating disk to v%d format\r\n", U3D_VER1); + + // create first epoch directory "0i0" + c3_c epo_c[8193]; + snprintf(epo_c, sizeof(epo_c), "%s/0i0", log_u->com_u->pax_c); + c3_mkdir(epo_c, 0700); + + // create epoch version file + c3_c epv_c[8193]; + snprintf(epv_c, sizeof(epv_c), "%s/epoc.txt", epo_c); + FILE* epv_f = fopen(epv_c, "a"); + fprintf(epv_f, "%d\n", U3D_VER1); + fclose(epv_f); + + // create binary version file + c3_c biv_c[8193]; + snprintf(biv_c, sizeof(biv_c), "%s/vere.txt", epo_c); + FILE* biv_f = fopen(biv_c, "a"); + fprintf(biv_f, URBIT_VERSION); + fclose(biv_f); + } + + // migrate existing unversioned pier to v1 + // + // XX write this + + return; +} \ No newline at end of file diff --git a/pkg/vere/main.c b/pkg/vere/main.c index 510ed95498..d2450b1cd9 100644 --- a/pkg/vere/main.c +++ b/pkg/vere/main.c @@ -2072,7 +2072,7 @@ _cw_chop(c3_i argc, c3_c* argv[]) // gracefully shutdown the pier if it's running u3_disk* old_u = _cw_disk_init(u3_Host.dir_c); - // note: this include patch applications (if any) + // note: this includes patch applications (if any) u3m_boot(u3_Host.dir_c, (size_t)1 << u3_Host.ops_u.lom_y); // check if there's a *current* snapshot @@ -2083,7 +2083,7 @@ _cw_chop(c3_i argc, c3_c* argv[]) exit(1); } - if ( c3n == u3e_backup(c3y)) { // backup current snapshot + if ( c3n == u3e_backup(c3y) ) { // backup current snapshot fprintf(stderr, "chop: error: failed to backup snapshot\r\n"); exit(1); } diff --git a/pkg/vere/vere.h b/pkg/vere/vere.h index a1522c6e40..a32d9a4604 100644 --- a/pkg/vere/vere.h +++ b/pkg/vere/vere.h @@ -8,6 +8,7 @@ #include "noun.h" #include "serf.h" #include "uv.h" +#include "version.h" /** Quasi-tunable parameters. **/ @@ -981,6 +982,16 @@ */ void u3_disk_plan(u3_disk* log_u, u3_fact* tac_u); + + /* u3_disk_last_epoc(): get latest epoch number. + */ + c3_d + u3_disk_last_epoc(u3_disk* log_u, c3_d* lat_d); + + /* u3_disk_migrate(): migrates disk format. + */ + void + u3_disk_migrate(u3_disk* log_u); /* u3_lord_init(): start serf. */ From 796de555b8f389961dba300d6dba5be027bc1fe3 Mon Sep 17 00:00:00 2001 From: Matthew LeVan Date: Fri, 14 Apr 2023 14:19:48 -0400 Subject: [PATCH 015/271] u3: make `u3e_backup` copy snapshot to arbitrary path --- pkg/noun/events.c | 38 ++++++++++++++++++++------------------ pkg/noun/events.h | 4 ++-- 2 files changed, 22 insertions(+), 20 deletions(-) diff --git a/pkg/noun/events.c b/pkg/noun/events.c index 2d831ffef4..9d772faf9b 100644 --- a/pkg/noun/events.c +++ b/pkg/noun/events.c @@ -993,48 +993,48 @@ _ce_image_copy(u3e_image* fom_u, u3e_image* tou_u) return c3y; } -/* u3e_backup(); +/* u3e_backup(): copy snapshot to pax_c, overwrite optionally. */ c3_o -u3e_backup(c3_o ovw_o) +u3e_backup(c3_c* pax_c, c3_o ovw_o) { u3e_image nop_u = { .nam_c = "north", .pgs_w = 0 }; u3e_image sop_u = { .nam_c = "south", .pgs_w = 0 }; c3_i mod_i = O_RDWR | O_CREAT; - c3_c ful_c[8193]; - snprintf(ful_c, 8192, "%s/.urb/bhk", u3P.dir_c); + if ( !pax_c ) { + fprintf(stderr, "loom: image backup: bad path\r\n"); + return c3n; + } - if ( (c3n == ovw_o) && c3_mkdir(ful_c, 0700) ) { + if ( (c3n == ovw_o) && c3_mkdir(pax_c, 0700) ) { if ( EEXIST != errno ) { fprintf(stderr, "loom: image backup: %s\r\n", strerror(errno)); } return c3n; } - snprintf(ful_c, 8192, "%s/.urb/bhk/%s.bin", u3P.dir_c, nop_u.nam_c); + c3_c nop_c[8193]; + snprintf(nop_c, 8192, "%s/%s.bin", pax_c, nop_u.nam_c); - if ( -1 == (nop_u.fid_i = c3_open(ful_c, mod_i, 0666)) ) { - fprintf(stderr, "loom: c3_open %s: %s\r\n", ful_c, strerror(errno)); + if ( -1 == (nop_u.fid_i = c3_open(nop_c, mod_i, 0666)) ) { + fprintf(stderr, "loom: c3_open %s: %s\r\n", nop_c, strerror(errno)); return c3n; } - snprintf(ful_c, 8192, "%s/.urb/bhk/%s.bin", u3P.dir_c, sop_u.nam_c); + c3_c sop_c[8193]; + snprintf(sop_c, 8192, "%s/%s.bin", pax_c, sop_u.nam_c); - if ( -1 == (sop_u.fid_i = c3_open(ful_c, mod_i, 0666)) ) { - fprintf(stderr, "loom: c3_open %s: %s\r\n", ful_c, strerror(errno)); + if ( -1 == (sop_u.fid_i = c3_open(sop_c, mod_i, 0666)) ) { + fprintf(stderr, "loom: c3_open %s: %s\r\n", sop_c, strerror(errno)); return c3n; } if ( (c3n == _ce_image_copy(&u3P.nor_u, &nop_u)) || (c3n == _ce_image_copy(&u3P.sou_u, &sop_u)) ) { - - c3_unlink(ful_c); - snprintf(ful_c, 8192, "%s/.urb/bhk/%s.bin", u3P.dir_c, nop_u.nam_c); - c3_unlink(ful_c); - snprintf(ful_c, 8192, "%s/.urb/bhk", u3P.dir_c); - c3_rmdir(ful_c); + c3_unlink(nop_c); + c3_unlink(sop_c); fprintf(stderr, "loom: image backup failed\r\n"); return c3n; } @@ -1111,7 +1111,9 @@ u3e_save(void) _ce_patch_free(pat_u); _ce_patch_delete(); - u3e_backup(c3n); + c3_c bhk_c[8193]; + snprintf(bhk_c, sizeof(bhk_c), "%s/.urb/bhk", u3P.dir_c); + u3e_backup(bhk_c, c3n); } /* u3e_live(): start the checkpointing system. diff --git a/pkg/noun/events.h b/pkg/noun/events.h index b4c78fe364..a4d7c5cf43 100644 --- a/pkg/noun/events.h +++ b/pkg/noun/events.h @@ -64,10 +64,10 @@ /** Functions. **/ - /* u3e_backup(): copy the snapshot from chk to bhk. + /* u3e_backup(): copy the snapshot to pax_c, overwrite optionally. */ c3_o - u3e_backup(c3_o ovw_o); + u3e_backup(c3_c* pax_c, c3_o ovw_o); /* u3e_fault(): handle a memory event with libsigsegv protocol. */ From 6d00b08ee3f6ffb94c1ca94ef2ad5870d7768c63 Mon Sep 17 00:00:00 2001 From: Matthew LeVan Date: Fri, 14 Apr 2023 14:22:33 -0400 Subject: [PATCH 016/271] disk: add `epoc` functions --- pkg/vere/disk.c | 140 +++++++++++++++++++++++++++++++++++------------- pkg/vere/vere.h | 10 ++-- 2 files changed, 110 insertions(+), 40 deletions(-) diff --git a/pkg/vere/disk.c b/pkg/vere/disk.c index 867716ac74..fee6ce28dd 100644 --- a/pkg/vere/disk.c +++ b/pkg/vere/disk.c @@ -893,24 +893,22 @@ u3_disk_init(c3_c* pax_c, u3_disk_cb cb_u) // validation before migration // - // migrate to the correct disk format + // migrate to the correct disk format if necessary // u3_disk_migrate(log_u); - // find the latest epoch directory + // initialize latest epoch // c3_d lat_d; - u3_disk_last_epoc(log_u, &lat_d); + u3_disk_epoc_last(log_u, &lat_d); fprintf(stderr, "disk: latest epoch is 0i%" PRIu64 "\r\n", lat_d); - // load latest epoch directory and set it in log_u + // initialize epoch's db // c3_c epo_c[8193]; snprintf(epo_c, 8192, "%s/0i%" PRIu64, log_c, lat_d); - log_u->epo_u = u3_foil_folder(epo_c); - - // initialize db of latest epoch - // + fprintf(stderr, "epo_c: %s\r\n", epo_c); + c3_free(log_c); { const size_t siz_i = // 500 GiB is as large as musl on aarch64 wants to allow @@ -919,15 +917,13 @@ u3_disk_init(c3_c* pax_c, u3_disk_cb cb_u) #else 0x10000000000; #endif - + if ( 0 == (log_u->mdb_u = u3_lmdb_init(epo_c, siz_i)) ) { fprintf(stderr, "disk: failed to initialize database\r\n"); c3_free(log_u); return 0; } } - - c3_free(log_c); } // get the latest event number from the db @@ -954,18 +950,101 @@ u3_disk_init(c3_c* pax_c, u3_disk_cb cb_u) return log_u; } -/* u3_disk_last_epoc: get latest epoch number. - */ -c3_d u3_disk_last_epoc(u3_disk* log_u, c3_d* lat_d) { - c3_d ret_d = 1; // return code 1 if no epoch directories exist - *lat_d = 0; // initialize lat_d to 0 +/* u3_disk_epoc_init: create new epoch. +*/ +c3_o u3_disk_epoc_init(u3_disk* log_u) { + // set new epoch number + c3_d lat_d, new_d; + c3_o eps_o = u3_disk_epoc_last(log_u, &lat_d); + if ( c3n == eps_o ) { + new_d = 0; // no epochs yet, so create first one 0i0 + } else { + // ensure latest epoch isn't empty + c3_d fir_d, las_d; + if ( c3n == u3_lmdb_gulf(log_u->mdb_u, &fir_d, &las_d) ) { + fprintf(stderr, "disk: failed to get first/last event numbers\r\n"); + return c3n; + } + if ( fir_d == las_d ) { + fprintf(stderr, "disk: latest epoch is empty; skipping epoch init\r\n"); + return c3n; + } + new_d = 1 + lat_d; // create next epoch + } + + // create new epoch directory + c3_c epo_c[8193]; + snprintf(epo_c, sizeof(epo_c), "%s/0i%" PRIu64, log_u->com_u->pax_c, new_d); + fprintf(stderr, "disk: creating new epoch directory 0i%" PRIu64 "\r\n", new_d); + if ( 0 != c3_mkdir(epo_c, 0700) ) { + fprintf(stderr, "disk: failed to create epoch directory\r\n"); + return c3n; + } + + // create epoch version file + c3_c epv_c[8193]; + snprintf(epv_c, sizeof(epv_c), "%s/epoc.txt", epo_c); + FILE* epv_f = fopen(epv_c, "a"); + fprintf(epv_f, "%d\n", U3D_VER1); + fclose(epv_f); + + // create binary version file + c3_c biv_c[8193]; + snprintf(biv_c, sizeof(biv_c), "%s/vere.txt", epo_c); + FILE* biv_f = fopen(biv_c, "a"); + fprintf(biv_f, URBIT_VERSION); + fclose(biv_f); + + // copy snapshot (skip if first epoch 0i0) + if ( new_d > 0 ) { + if ( c3n == u3e_backup(epo_c, c3n) ) { + fprintf(stderr, "disk: failed to copy snapshot to new epoch\r\n"); + goto fail; + } + } + + // initialize db of latest epoch + { + const size_t siz_i = + // 500 GiB is as large as musl on aarch64 wants to allow + #if (defined(U3_CPU_aarch64) && defined(U3_OS_linux)) + 0x7d00000000; + #else + 0x10000000000; + #endif + + if ( 0 == (log_u->mdb_u = u3_lmdb_init(epo_c, siz_i)) ) { + fprintf(stderr, "disk: failed to initialize database\r\n"); + c3_free(log_u); + goto fail; + } + } + + // load new epoch directory and set it in log_u + log_u->epo_u = u3_foil_folder(epo_c); + + // success + return c3y; + +fail: + c3_unlink(epv_c); + c3_unlink(biv_c); + c3_rmdir(epo_c); + return c3n; +} + +/* u3_disk_epoc_last: get latest epoch number. +*/ +c3_o u3_disk_epoc_last(u3_disk* log_u, c3_d* lat_d) { + c3_o ret_d = c3n; // return no if no epoch directories exist + *lat_d = 0; // initialize lat_d to 0 u3_dent* den_u = log_u->com_u->dir_u; while ( den_u ) { c3_d epo_d = 0; if ( 1 != sscanf(den_u->nam_c, "0i%" PRIu64, &epo_d) ) { fprintf(stderr, "disk: epoch directory is not a @ui: %s\r\n", den_u->nam_c); } else { - ret_d = 0; // return code 0 if at least one epoch directory exists + ret_d = c3y; // return yes if at least one epoch directory exists } *lat_d = c3_max(epo_d, *lat_d); // update the latest epoch number den_u = den_u->nex_u; @@ -982,30 +1061,17 @@ void u3_disk_migrate(u3_disk* log_u) // initialize disk v1 on fresh boots // - if ( 1 == u3_disk_last_epoc(log_u, &lat_d) ) { // XX add another condition to ensure fresh boot + if ( c3n == u3_disk_epoc_last(log_u, &lat_d) ) { // XX ensure fresh boot fprintf(stderr, "disk: migrating disk to v%d format\r\n", U3D_VER1); - // create first epoch directory "0i0" - c3_c epo_c[8193]; - snprintf(epo_c, sizeof(epo_c), "%s/0i0", log_u->com_u->pax_c); - c3_mkdir(epo_c, 0700); - - // create epoch version file - c3_c epv_c[8193]; - snprintf(epv_c, sizeof(epv_c), "%s/epoc.txt", epo_c); - FILE* epv_f = fopen(epv_c, "a"); - fprintf(epv_f, "%d\n", U3D_VER1); - fclose(epv_f); - - // create binary version file - c3_c biv_c[8193]; - snprintf(biv_c, sizeof(biv_c), "%s/vere.txt", epo_c); - FILE* biv_f = fopen(biv_c, "a"); - fprintf(biv_f, URBIT_VERSION); - fclose(biv_f); + // initialize first epoch "0i0" + if ( c3n == u3_disk_epoc_init(log_u) ) { + fprintf(stderr, "disk: migrate: failed to initialize first epoch\r\n"); + exit(1); + } } - // migrate existing unversioned pier to v1 + // migrate existing, unversioned disk to v1 // // XX write this diff --git a/pkg/vere/vere.h b/pkg/vere/vere.h index a32d9a4604..694769cf57 100644 --- a/pkg/vere/vere.h +++ b/pkg/vere/vere.h @@ -8,7 +8,6 @@ #include "noun.h" #include "serf.h" #include "uv.h" -#include "version.h" /** Quasi-tunable parameters. **/ @@ -983,10 +982,15 @@ void u3_disk_plan(u3_disk* log_u, u3_fact* tac_u); + /* u3_disk_init_epoc(): create new epoch. + */ + c3_o + u3_disk_epoc_init(u3_disk* log_u); + /* u3_disk_last_epoc(): get latest epoch number. */ - c3_d - u3_disk_last_epoc(u3_disk* log_u, c3_d* lat_d); + c3_o + u3_disk_epoc_last(u3_disk* log_u, c3_d* lat_d); /* u3_disk_migrate(): migrates disk format. */ From 488adc3f7cede004a09b18796386ae03409b6d3c Mon Sep 17 00:00:00 2001 From: Matthew LeVan Date: Mon, 17 Apr 2023 05:13:11 -0400 Subject: [PATCH 017/271] cli: add `roll` subcommand for epoch rollovers --- pkg/noun/events.h | 2 +- pkg/vere/disk.c | 72 +++++++++++++++++++++++++++------------- pkg/vere/foil.c | 4 +-- pkg/vere/main.c | 84 +++++++++++++++++++++++++++++++++++++++++++++-- pkg/vere/vere.h | 2 +- pkg/vere/ward.c | 12 +++++++ 6 files changed, 147 insertions(+), 29 deletions(-) diff --git a/pkg/noun/events.h b/pkg/noun/events.h index a4d7c5cf43..058fa2fe23 100644 --- a/pkg/noun/events.h +++ b/pkg/noun/events.h @@ -64,7 +64,7 @@ /** Functions. **/ - /* u3e_backup(): copy the snapshot to pax_c, overwrite optionally. + /* u3e_backup(): copy the snapshot to pax_c, overwrite optional. */ c3_o u3e_backup(c3_c* pax_c, c3_o ovw_o); diff --git a/pkg/vere/disk.c b/pkg/vere/disk.c index fee6ce28dd..8eefc7c973 100644 --- a/pkg/vere/disk.c +++ b/pkg/vere/disk.c @@ -890,26 +890,25 @@ u3_disk_init(c3_c* pax_c, u3_disk_cb cb_u) return 0; } - // validation before migration + // XX validation before migration // // migrate to the correct disk format if necessary // u3_disk_migrate(log_u); - // initialize latest epoch + // get latest epoch number // c3_d lat_d; u3_disk_epoc_last(log_u, &lat_d); - fprintf(stderr, "disk: latest epoch is 0i%" PRIu64 "\r\n", lat_d); // initialize epoch's db // c3_c epo_c[8193]; snprintf(epo_c, 8192, "%s/0i%" PRIu64, log_c, lat_d); - fprintf(stderr, "epo_c: %s\r\n", epo_c); c3_free(log_c); { + // XX extract into function? const size_t siz_i = // 500 GiB is as large as musl on aarch64 wants to allow #if (defined(U3_CPU_aarch64) && defined(U3_OS_linux)) @@ -924,24 +923,33 @@ u3_disk_init(c3_c* pax_c, u3_disk_cb cb_u) return 0; } } - } + fprintf(stderr, "disk: loaded epoch 0i%" PRIu64 "\r\n", lat_d); - // get the latest event number from the db - // - { - log_u->dun_d = 0; - c3_d fir_d; - - if ( c3n == u3_lmdb_gulf(log_u->mdb_u, &fir_d, &log_u->dun_d) ) { - fprintf(stderr, "disk: failed to load latest event from database\r\n"); - c3_free(log_u); + // get first/last event numbers from lmdb + c3_d fir_d, las_d; + if ( c3n == u3_lmdb_gulf(log_u->mdb_u, &fir_d, &las_d) ) { + fprintf(stderr, "disk: failed to get first/last event numbers\r\n"); return 0; } - log_u->sen_d = log_u->dun_d; + // initialize dun_d/sen_d values + if ( 0 == las_d ) { // fresh epoch (no events in lmdb yet) + if ( 0 == lat_d ) { // first epoch + log_u->dun_d = 0; + log_u->sen_d = 0; + } else { // not first epoch + log_u->dun_d = lat_d - 1; // set dun_d to last event in prev epoch + log_u->sen_d = log_u->dun_d; + } + } else { // not fresh epoch + log_u->dun_d = las_d; // set dun_d to last event in lmdb + log_u->sen_d = las_d; + } + + // mark the log as live + log_u->liv_o = c3y; } - log_u->liv_o = c3y; #if defined(DISK_TRACE_JAM) || defined(DISK_TRACE_CUE) u3t_trace_open(pax_c); @@ -966,16 +974,15 @@ c3_o u3_disk_epoc_init(u3_disk* log_u) { return c3n; } if ( fir_d == las_d ) { - fprintf(stderr, "disk: latest epoch is empty; skipping epoch init\r\n"); + fprintf(stderr, "disk: latest epoch is empty; skipping rollover\r\n"); return c3n; } - new_d = 1 + lat_d; // create next epoch + new_d = 1 + las_d; // create next epoch } // create new epoch directory c3_c epo_c[8193]; snprintf(epo_c, sizeof(epo_c), "%s/0i%" PRIu64, log_u->com_u->pax_c, new_d); - fprintf(stderr, "disk: creating new epoch directory 0i%" PRIu64 "\r\n", new_d); if ( 0 != c3_mkdir(epo_c, 0700) ) { fprintf(stderr, "disk: failed to create epoch directory\r\n"); return c3n; @@ -995,15 +1002,26 @@ c3_o u3_disk_epoc_init(u3_disk* log_u) { fprintf(biv_f, URBIT_VERSION); fclose(biv_f); - // copy snapshot (skip if first epoch 0i0) + // copy snapshot (skip if first epoch) if ( new_d > 0 ) { - if ( c3n == u3e_backup(epo_c, c3n) ) { + if ( c3n == u3e_backup(epo_c, c3y) ) { fprintf(stderr, "disk: failed to copy snapshot to new epoch\r\n"); goto fail; } } - // initialize db of latest epoch + // get metadata from old epoch's db + c3_d who_d[2]; + c3_o fak_o; + c3_w lif_w; + if ( c3y == eps_o ) { + if ( c3y != u3_disk_read_meta(log_u->mdb_u, who_d, &fak_o, &lif_w) ) { + fprintf(stderr, "disk: failed to read metadata\r\n"); + goto fail; + } + } + + // initialize db of new epoch { const size_t siz_i = // 500 GiB is as large as musl on aarch64 wants to allow @@ -1020,6 +1038,14 @@ c3_o u3_disk_epoc_init(u3_disk* log_u) { } } + // write the metadata to the database + if ( c3y == eps_o ) { + if ( c3n == u3_disk_save_meta(log_u->mdb_u, who_d, fak_o, lif_w) ) { + fprintf(stderr, "disk: failed to save metadata\r\n"); + exit(1); + } + } + // load new epoch directory and set it in log_u log_u->epo_u = u3_foil_folder(epo_c); @@ -1038,7 +1064,7 @@ c3_o u3_disk_epoc_init(u3_disk* log_u) { c3_o u3_disk_epoc_last(u3_disk* log_u, c3_d* lat_d) { c3_o ret_d = c3n; // return no if no epoch directories exist *lat_d = 0; // initialize lat_d to 0 - u3_dent* den_u = log_u->com_u->dir_u; + u3_dent* den_u = log_u->com_u->dil_u; while ( den_u ) { c3_d epo_d = 0; if ( 1 != sscanf(den_u->nam_c, "0i%" PRIu64, &epo_d) ) { diff --git a/pkg/vere/foil.c b/pkg/vere/foil.c index a241c32b66..9fae86973f 100644 --- a/pkg/vere/foil.c +++ b/pkg/vere/foil.c @@ -115,8 +115,8 @@ u3_foil_folder(const c3_c* pax_c) dir_u->all_u = det_u; } else if ( UV_DIRENT_DIR == den_u.type ) { u3_dent* det_u = u3_dent_init(den_u.name); - det_u->nex_u = dir_u->dir_u; - dir_u->dir_u = det_u; + det_u->nex_u = dir_u->dil_u; + dir_u->dil_u = det_u; } } diff --git a/pkg/vere/main.c b/pkg/vere/main.c index d2450b1cd9..6aba090477 100644 --- a/pkg/vere/main.c +++ b/pkg/vere/main.c @@ -682,6 +682,7 @@ _cw_usage(c3_c* bin_c) " %s next %.*s request upgrade:\n", " %s queu %.*s cue state:\n", " %s chop %.*s truncate event log:\n", + " %s roll %.*s rollover to new epoch:\n", " %s vere ARGS download binary:\n", "\n run as a 'serf':\n", " %s serf " @@ -1151,7 +1152,7 @@ _cw_disk_init(c3_c* dir_c) u3_disk* log_u = u3_disk_init(dir_c, cb_u); if ( !log_u ) { - fprintf(stderr, "unable to open event log\n"); + fprintf(stderr, "unable to open event log\n");; exit(1); } @@ -2083,7 +2084,9 @@ _cw_chop(c3_i argc, c3_c* argv[]) exit(1); } - if ( c3n == u3e_backup(c3y) ) { // backup current snapshot + c3_c bhk_c[8193]; + snprintf(bhk_c, sizeof(bhk_c), "%s/.urb/bhk", u3_Host.dir_c); + if ( c3n == u3e_backup(bhk_c, c3y) ) { // backup current snapshot fprintf(stderr, "chop: error: failed to backup snapshot\r\n"); exit(1); } @@ -2189,6 +2192,82 @@ _cw_chop(c3_i argc, c3_c* argv[]) fprintf(stderr, " `mv %s %s` then try again\r\n", bak_c, dat_c); } +/* _cw_roll(): rollover to new epoch + */ +static void +_cw_roll(c3_i argc, c3_c* argv[]) +{ + c3_i ch_i, lid_i; + c3_w arg_w; + + static struct option lop_u[] = { + { "loom", required_argument, NULL, c3__loom }, + { NULL, 0, NULL, 0 } + }; + + u3_Host.dir_c = _main_pier_run(argv[0]); + + while ( -1 != (ch_i=getopt_long(argc, argv, "", lop_u, &lid_i)) ) { + switch ( ch_i ) { + case c3__loom: { + if (_main_readw_loom("loom", &u3_Host.ops_u.lom_y)) { + exit(1); + } + } break; + + case '?': { + fprintf(stderr, "invalid argument\r\n"); + exit(1); + } break; + } + } + + // argv[optind] is always "roll" + // + + if ( !u3_Host.dir_c ) { + if ( optind + 1 < argc ) { + u3_Host.dir_c = argv[optind + 1]; + } + else { + fprintf(stderr, "invalid command, pier required\r\n"); + exit(1); + } + + optind++; + } + + if ( optind + 1 != argc ) { + fprintf(stderr, "invalid command\r\n"); + exit(1); + } + + // gracefully shutdown the pier if it's running + u3_disk* log_u = _cw_disk_init(u3_Host.dir_c); + + // note: this includes patch applications (if any) + u3m_boot(u3_Host.dir_c, (size_t)1 << u3_Host.ops_u.lom_y); + + // check if there's a *current* snapshot + if ( log_u->dun_d != u3A->eve_d ) { + fprintf(stderr, "roll: error: snapshot is out of date, please " + "start/shutdown your pier gracefully first\r\n"); + fprintf(stderr, "roll: eve_d: %" PRIu64 ", dun_d: %" PRIu64 "\r\n", \ + u3A->eve_d, log_u->dun_d); + exit(1); + } + + // create new epoch + if ( c3n == u3_disk_epoc_init(log_u) ) { + fprintf(stderr, "roll: error: failed to initialize new epoch\r\n"); + exit(1); + } + + // report success + c3_d epo_d = log_u->dun_d + 1; + fprintf(stderr, "roll: success: created epoch 0i%" PRIu64 "\r\n", epo_d); +} + /* _cw_vere(): download vere */ static void @@ -2451,6 +2530,7 @@ _cw_utils(c3_i argc, c3_c* argv[]) case c3__prep: _cw_prep(argc, argv); return 2; // continue on case c3__queu: _cw_queu(argc, argv); return 1; case c3__chop: _cw_chop(argc, argv); return 1; + case c3__roll: _cw_roll(argc, argv); return 1; case c3__vere: _cw_vere(argc, argv); return 1; case c3__vile: _cw_vile(argc, argv); return 1; diff --git a/pkg/vere/vere.h b/pkg/vere/vere.h index 694769cf57..fde11bb193 100644 --- a/pkg/vere/vere.h +++ b/pkg/vere/vere.h @@ -130,7 +130,7 @@ c3_c* pax_c; // path of directory uv_file fil_u; // file, opened read-only to fsync u3_dent* all_u; // file list - u3_dent* dir_u; // directory list + u3_dent* dil_u; // directory list } u3_dire; /* u3_save: checkpoint control. diff --git a/pkg/vere/ward.c b/pkg/vere/ward.c index 4c90be7179..9d6c8cbed3 100644 --- a/pkg/vere/ward.c +++ b/pkg/vere/ward.c @@ -39,6 +39,7 @@ u3_dire_init(const c3_c* pax_c) { u3_dire *dir_u = c3_malloc(sizeof *dir_u); dir_u->all_u = 0; + dir_u->dil_u = 0; dir_u->pax_c = c3_malloc(1 + strlen(pax_c)); strcpy(dir_u->pax_c, pax_c); @@ -61,6 +62,17 @@ u3_dire_free(u3_dire *dir_u) } } + { + u3_dent *det_u = dir_u->dil_u; + u3_dent *nex_u; + + while ( det_u ) { + nex_u = det_u->nex_u; + u3_dent_free(det_u); + det_u = nex_u; + } + } + c3_free(dir_u->pax_c); c3_free(dir_u); } From f0289f989d350692246d29a00a1694de68677659 Mon Sep 17 00:00:00 2001 From: Matthew LeVan Date: Mon, 17 Apr 2023 12:45:38 -0400 Subject: [PATCH 018/271] cli: make `chop` work with epochs --- pkg/vere/disk.c | 30 ++++++++++++ pkg/vere/main.c | 118 ++++++++++-------------------------------------- pkg/vere/vere.h | 9 +++- 3 files changed, 60 insertions(+), 97 deletions(-) diff --git a/pkg/vere/disk.c b/pkg/vere/disk.c index 8eefc7c973..9aa0f68707 100644 --- a/pkg/vere/disk.c +++ b/pkg/vere/disk.c @@ -1059,6 +1059,36 @@ c3_o u3_disk_epoc_init(u3_disk* log_u) { return c3n; } +/* u3_disk_epoc_kill: delete an epoch. +*/ +c3_o u3_disk_epoc_kill(u3_disk* log_u, c3_d epo_d) { + // get epoch directory + c3_c epo_c[8193]; + snprintf(epo_c, sizeof(epo_c), "%s/0i%" PRIu64, log_u->com_u->pax_c, epo_d); + + // delete files in epoch directory + u3_dire* dir_u = u3_foil_folder(epo_c); + u3_dent* den_u = dir_u->all_u; + while ( den_u ) { + c3_c fil_c[8193]; + snprintf(fil_c, sizeof(fil_c), "%s/%s", epo_c, den_u->nam_c); + if ( 0 != c3_unlink(fil_c) ) { + fprintf(stderr, "disk: failed to delete file in epoch directory\r\n"); + return c3n; + } + den_u = den_u->nex_u; + } + + // delete epoch directory + if ( 0 != c3_rmdir(epo_c) ) { + fprintf(stderr, "disk: failed to delete epoch directory\r\n"); + return c3n; + } + + // success + return c3y; +} + /* u3_disk_epoc_last: get latest epoch number. */ c3_o u3_disk_epoc_last(u3_disk* log_u, c3_d* lat_d) { diff --git a/pkg/vere/main.c b/pkg/vere/main.c index 6aba090477..334a9dddd6 100644 --- a/pkg/vere/main.c +++ b/pkg/vere/main.c @@ -2071,125 +2071,53 @@ _cw_chop(c3_i argc, c3_c* argv[]) } // gracefully shutdown the pier if it's running - u3_disk* old_u = _cw_disk_init(u3_Host.dir_c); + u3_disk* log_u = _cw_disk_init(u3_Host.dir_c); // note: this includes patch applications (if any) u3m_boot(u3_Host.dir_c, (size_t)1 << u3_Host.ops_u.lom_y); // check if there's a *current* snapshot - if ( old_u->dun_d != u3A->eve_d ) { + if ( log_u->dun_d != u3A->eve_d ) { fprintf(stderr, "chop: error: snapshot is out of date, please " "start/shutdown your pier gracefully first\r\n"); - fprintf(stderr, "chop: eve_d: %" PRIu64 ", dun_d: %" PRIu64 "\r\n", u3A->eve_d, old_u->dun_d); - exit(1); - } - - c3_c bhk_c[8193]; - snprintf(bhk_c, sizeof(bhk_c), "%s/.urb/bhk", u3_Host.dir_c); - if ( c3n == u3e_backup(bhk_c, c3y) ) { // backup current snapshot - fprintf(stderr, "chop: error: failed to backup snapshot\r\n"); - exit(1); - } - - // initialize the lmdb environment - // see disk.c:885 - const size_t siz_i = - // 500 GiB is as large as musl on aarch64 wants to allow - #if (defined(U3_CPU_aarch64) && defined(U3_OS_linux)) - 0x7d00000000; - #else - 0x10000000000; - #endif - c3_c log_c[8193]; - snprintf(log_c, sizeof(log_c), "%s/.urb/log", u3_Host.dir_c); - - // get the first/last event numbers from the event log - c3_d fir_d, las_d; - if ( c3n == u3_lmdb_gulf(old_u->mdb_u, &fir_d, &las_d) ) { - fprintf(stderr, "chop: failed to load latest event from database\r\n"); + fprintf(stderr, "chop: eve_d: %" PRIu64 ", dun_d: %" PRIu64 "\r\n", u3A->eve_d, log_u->dun_d); exit(1); } - // get the metadata - c3_d who_d[2]; - c3_o fak_o; - c3_w lif_w; - if ( c3y != u3_disk_read_meta(old_u->mdb_u, who_d, &fak_o, &lif_w) ) { - fprintf(stderr, "chop: failed to read metadata\r\n"); + // create new epoch + if ( c3n == u3_disk_epoc_init(log_u) ) { + fprintf(stderr, "chop: failed to create new epoch\r\n"); exit(1); } - // get the last event - u3_lmdb_walk itr_u; - size_t len_i; - void* buf_v[1]; - if ( c3n == u3_lmdb_walk_init(old_u->mdb_u, &itr_u, las_d, las_d) ) { - fprintf(stderr, "chop: failed to initialize iterator\r\n"); - exit(1); - } - if ( c3n == u3_lmdb_walk_next(&itr_u, &len_i, buf_v) ) { - fprintf(stderr, "chop: failed to read event\r\n"); + // get latest epoch number + c3_d lat_d; + if ( c3n == u3_disk_epoc_last(log_u, &lat_d) ) { + fprintf(stderr, "chop: failed to find last epoch\r\n"); exit(1); } - u3_lmdb_walk_done(&itr_u); - // initialize a fresh lmdb environment in the "chop" subdir - c3_c cho_c[8193]; - snprintf(cho_c, sizeof(cho_c), "%s/chop", log_c); - if ( 0 != access(cho_c, F_OK) ) { - if ( 0 != c3_mkdir(cho_c, 0700) ) { - fprintf(stderr, "chop: failed to create chop directory\r\n"); - exit(1); + // delete all but the last epoch + u3_dent* den_u = log_u->com_u->dil_u; + c3_d epo_d = 0; + while ( den_u && epo_d <= lat_d ) { + if ( 1 != sscanf(den_u->nam_c, "0i%" PRIu64, &epo_d) ) { + fprintf(stderr, "disk: epoch directory is not a @ui: %s\r\n", den_u->nam_c); + } else { + fprintf(stderr, "chop: deleting epoch 0i%" PRIu64 "\r\n", epo_d); + if ( c3y != u3_disk_epoc_kill(log_u, epo_d) ) { + fprintf(stderr, "chop: failed to delete epoch 0i%" PRIu64 "\r\n", epo_d); + } } - } - MDB_env* new_u = u3_lmdb_init(cho_c, siz_i); - if ( !new_u ) { - fprintf(stderr, "chop: failed to initialize new database\r\n"); - exit(1); - } - - // write the metadata to the database - if ( c3n == u3_disk_save_meta(new_u, who_d, fak_o, lif_w) ) { - fprintf(stderr, "chop: failed to save metadata\r\n"); - exit(1); - } - - // write the last event to the database - // warning: this relies on the old database still being open - if ( c3n == u3_lmdb_save(new_u, las_d, 1, buf_v, &len_i) ) { - fprintf(stderr, "chop: failed to write last event\r\n"); - exit(1); - } - - // backup the original database file - c3_c dat_c[8193], bak_c[8193]; - snprintf(dat_c, sizeof(dat_c), "%s/data.mdb", log_c); - // "data_-.mdb.bak" - snprintf(bak_c, sizeof(bak_c), "%s/data_%" PRIu64 "-%" PRIu64 ".mdb.bak", cho_c, fir_d, las_d); - if ( 0 != c3_rename(dat_c, bak_c) ) { - fprintf(stderr, "chop: failed to backup original database file\r\n"); - exit(1); - } - - // rename new database file to be official - c3_c new_c[8193]; - snprintf(new_c, sizeof(new_c), "%s/data.mdb", cho_c); - if ( 0 != c3_rename(new_c, dat_c) ) { - fprintf(stderr, "chop: failed to rename new database file\r\n"); - exit(1); + den_u = den_u->nex_u; } // cleanup - u3_disk_exit(old_u); - u3_lmdb_exit(new_u); + u3_disk_exit(log_u); u3m_stop(); // success fprintf(stderr, "chop: event log truncation complete\r\n"); - fprintf(stderr, " event log backup written to %s\r\n", bak_c); - fprintf(stderr, " WARNING: ENSURE YOU CAN RESTART YOUR SHIP BEFORE DELETING YOUR EVENT LOG BACKUP FILE!\r\n"); - fprintf(stderr, " if you can't, restore your log by running:\r\n"); - fprintf(stderr, " `mv %s %s` then try again\r\n", bak_c, dat_c); } /* _cw_roll(): rollover to new epoch diff --git a/pkg/vere/vere.h b/pkg/vere/vere.h index fde11bb193..a2726d2341 100644 --- a/pkg/vere/vere.h +++ b/pkg/vere/vere.h @@ -982,12 +982,17 @@ void u3_disk_plan(u3_disk* log_u, u3_fact* tac_u); - /* u3_disk_init_epoc(): create new epoch. + /* u3_disk_epoc_init(): create new epoch. */ c3_o u3_disk_epoc_init(u3_disk* log_u); - /* u3_disk_last_epoc(): get latest epoch number. + /* u3_disk_epoc_kill(): delete an epoch. + */ + c3_o + u3_disk_epoc_kill(u3_disk* log_u, c3_d epo_d); + + /* u3_disk_epoc_last(): get latest epoch number. */ c3_o u3_disk_epoc_last(u3_disk* log_u, c3_d* lat_d); From c54d6c0bba8dfa100705322a40536dd4695e1499 Mon Sep 17 00:00:00 2001 From: Matthew LeVan Date: Tue, 18 Apr 2023 15:23:03 -0400 Subject: [PATCH 019/271] disk: migrate existing piers to v1 format (epochs) --- pkg/vere/disk.c | 123 ++++++++++++++++++++++++++++++++++++++++++------ pkg/vere/main.c | 6 +-- 2 files changed, 112 insertions(+), 17 deletions(-) diff --git a/pkg/vere/disk.c b/pkg/vere/disk.c index 9aa0f68707..72dd3fcf91 100644 --- a/pkg/vere/disk.c +++ b/pkg/vere/disk.c @@ -891,19 +891,15 @@ u3_disk_init(c3_c* pax_c, u3_disk_cb cb_u) } // XX validation before migration - // // migrate to the correct disk format if necessary - // u3_disk_migrate(log_u); // get latest epoch number - // c3_d lat_d; u3_disk_epoc_last(log_u, &lat_d); // initialize epoch's db - // c3_c epo_c[8193]; snprintf(epo_c, 8192, "%s/0i%" PRIu64, log_c, lat_d); c3_free(log_c); @@ -984,7 +980,7 @@ c3_o u3_disk_epoc_init(u3_disk* log_u) { c3_c epo_c[8193]; snprintf(epo_c, sizeof(epo_c), "%s/0i%" PRIu64, log_u->com_u->pax_c, new_d); if ( 0 != c3_mkdir(epo_c, 0700) ) { - fprintf(stderr, "disk: failed to create epoch directory\r\n"); + fprintf(stderr, "disk: failed to create epoch directory %" PRIu64 "\r\n", new_d); return c3n; } @@ -1094,7 +1090,7 @@ c3_o u3_disk_epoc_kill(u3_disk* log_u, c3_d epo_d) { c3_o u3_disk_epoc_last(u3_disk* log_u, c3_d* lat_d) { c3_o ret_d = c3n; // return no if no epoch directories exist *lat_d = 0; // initialize lat_d to 0 - u3_dent* den_u = log_u->com_u->dil_u; + u3_dent* den_u = u3_foil_folder(log_u->com_u->pax_c)->dil_u; while ( den_u ) { c3_d epo_d = 0; if ( 1 != sscanf(den_u->nam_c, "0i%" PRIu64, &epo_d) ) { @@ -1113,23 +1109,122 @@ c3_o u3_disk_epoc_last(u3_disk* log_u, c3_d* lat_d) { */ void u3_disk_migrate(u3_disk* log_u) { - c3_d lat_d; // latest epoch number + // check if data.mdb exists in log directory + c3_o dat_o = c3n; + c3_c dat_c[8193]; + snprintf(dat_c, sizeof(dat_c), "%s/data.mdb", log_u->com_u->pax_c); + if ( 0 == access(dat_c, F_OK) ) { + dat_o = c3y; + } - // initialize disk v1 on fresh boots - // - if ( c3n == u3_disk_epoc_last(log_u, &lat_d) ) { // XX ensure fresh boot + // check if any epochs exist + c3_d lat_d; + c3_o epo_o = u3_disk_epoc_last(log_u, &lat_d); + + if ( c3n == epo_o && c3n == dat_o ) { + // initialize disk v1 on fresh boots + fprintf(stderr, "disk: initializing disk with v%d format\r\n", U3D_VER1); + + // initialize first epoch "0i0" + if ( c3n == u3_disk_epoc_init(log_u) ) { + fprintf(stderr, "disk: migrate: failed to initialize first epoch\r\n"); + exit(1); + } + } else if ( c3n == epo_o ) { + // migrate existing, unversioned disk to v1 fprintf(stderr, "disk: migrating disk to v%d format\r\n", U3D_VER1); + // initialize pre-migrated lmdb + MDB_env* old_u; + { + const size_t siz_i = + // 500 GiB is as large as musl on aarch64 wants to allow + #if (defined(U3_CPU_aarch64) && defined(U3_OS_linux)) + 0x7d00000000; + #else + 0x10000000000; + #endif + + if ( 0 == (old_u = u3_lmdb_init(log_u->com_u->pax_c, siz_i)) ) { + fprintf(stderr, "disk: failed to initialize database\r\n"); + c3_free(log_u); + return; + } + } + + // get first/last event numbers from pre-migrated lmdb + c3_d fir_d, las_d; + if ( c3n == u3_lmdb_gulf(old_u, &fir_d, &las_d) ) { + fprintf(stderr, "disk: failed to get first/last event numbers\r\n"); + return; + } + + // boot + u3m_boot(u3_Host.dir_c, (size_t)1 << u3_Host.ops_u.lom_y); + + // make sure there's a current snapshot + if ( las_d != u3A->eve_d ) { + fprintf(stderr, "disk: migrate: error: snapshot is out of date, please " + "start/shutdown your pier gracefully first\r\n"); + exit(1); + } + + // shutdown pre-migrated u3 system and lmdb + u3m_stop(); + // initialize first epoch "0i0" if ( c3n == u3_disk_epoc_init(log_u) ) { fprintf(stderr, "disk: migrate: failed to initialize first epoch\r\n"); exit(1); } - } - // migrate existing, unversioned disk to v1 - // - // XX write this + // move data.mdb to 0i0/data.mdb + c3_c new_c[8193]; + snprintf(new_c, sizeof(new_c), "%s/0i0/data.mdb", log_u->com_u->pax_c); + if ( 0 != rename(dat_c, new_c) ) { + fprintf(stderr, "disk: migrate: failed to move data.mdb\r\n"); + exit(1); + } + + // move lock.mdb to 0i0/lock.mdb + c3_c old_c[8193]; + snprintf(old_c, sizeof(old_c), "%s/lock.mdb", log_u->com_u->pax_c); + snprintf(new_c, sizeof(new_c), "%s/0i0/lock.mdb", log_u->com_u->pax_c); + if ( 0 != rename(old_c, new_c) ) { + fprintf(stderr, "disk: migrate: failed to move lock.mdb\r\n"); + exit(1); + } + + // rollover to new epoch + log_u->mdb_u = old_u; + if ( c3n == u3_disk_epoc_init(log_u) ) { + fprintf(stderr, "roll: error: failed to initialize new epoch\r\n"); + exit(1); + } + + // delete backup snapshot + c3_c bhk_c[8193]; + snprintf(bhk_c, sizeof(bhk_c), "%s/.urb/bhk", u3_Host.dir_c); + c3_c nop_c[8193]; + snprintf(nop_c, sizeof(nop_c), "%s/north.bin", bhk_c); + c3_c sop_c[8193]; + snprintf(sop_c, sizeof(sop_c), "%s/south.bin", bhk_c); + if ( c3n == c3_unlink(nop_c) ) { + fprintf(stderr, "disk: migrate: failed to delete north.bin\r\n"); + exit(1); + } else if ( c3n == c3_unlink(sop_c) ) { + fprintf(stderr, "disk: migrate: failed to delete south.bin\r\n"); + exit(1); + } else { + if ( c3n == c3_rmdir(bhk_c) ) { + fprintf(stderr, "disk: migrate: failed to delete bhk\r\n"); + exit(1); + } + } + + // success + fprintf(stderr, "disk: migrated disk to v%d format\r\n", U3D_VER1); + } return; } \ No newline at end of file diff --git a/pkg/vere/main.c b/pkg/vere/main.c index 334a9dddd6..c4fd9de29c 100644 --- a/pkg/vere/main.c +++ b/pkg/vere/main.c @@ -2097,7 +2097,7 @@ _cw_chop(c3_i argc, c3_c* argv[]) exit(1); } - // delete all but the last epoch + // delete all but the newly created epoch u3_dent* den_u = log_u->com_u->dil_u; c3_d epo_d = 0; while ( den_u && epo_d <= lat_d ) { @@ -2191,9 +2191,9 @@ _cw_roll(c3_i argc, c3_c* argv[]) exit(1); } - // report success + // success c3_d epo_d = log_u->dun_d + 1; - fprintf(stderr, "roll: success: created epoch 0i%" PRIu64 "\r\n", epo_d); + fprintf(stderr, "roll: epoch rollover complete"); } /* _cw_vere(): download vere From f8834e0871de94fbf2a55458b23f8c6cffb87fc0 Mon Sep 17 00:00:00 2001 From: Matthew LeVan Date: Wed, 26 Apr 2023 13:51:17 -0400 Subject: [PATCH 020/271] disk: validate current snapshot before migration --- pkg/vere/disk.c | 116 ++++++++++++++++++++++++------------------------ pkg/vere/main.c | 10 +++++ pkg/vere/vere.h | 3 +- 3 files changed, 70 insertions(+), 59 deletions(-) diff --git a/pkg/vere/disk.c b/pkg/vere/disk.c index 72dd3fcf91..bd6f8a4d27 100644 --- a/pkg/vere/disk.c +++ b/pkg/vere/disk.c @@ -878,31 +878,33 @@ u3_disk_init(c3_c* pax_c, u3_disk_cb cb_u) // XX closures? // { - c3_c* log_c = c3_malloc(10 + strlen(pax_c)); - - strcpy(log_c, pax_c); - strcat(log_c, "/.urb/log"); + c3_c log_c[8193]; + snprintf(log_c, sizeof(log_c), "%s/.urb/log", pax_c); if ( 0 == (log_u->com_u = u3_foil_folder(log_c)) ) { fprintf(stderr, "disk: failed to load /.urb/log in %s\r\n", pax_c); - c3_free(log_c); c3_free(log_u); return 0; } - // XX validation before migration - // migrate to the correct disk format if necessary - u3_disk_migrate(log_u); + if ( c3n == u3_disk_migrate(log_u) ) { + fprintf(stderr, "disk: failed to migrate to v%d\r\n", U3D_VER1); + c3_free(log_u); + return 0; + } // get latest epoch number c3_d lat_d; - u3_disk_epoc_last(log_u, &lat_d); + if ( c3n == u3_disk_epoc_last(log_u, &lat_d) ) { + fprintf(stderr, "disk: failed to load epoch number\r\n"); + c3_free(log_u); + return 0; + } // initialize epoch's db c3_c epo_c[8193]; snprintf(epo_c, 8192, "%s/0i%" PRIu64, log_c, lat_d); - c3_free(log_c); { // XX extract into function? const size_t siz_i = @@ -932,16 +934,15 @@ u3_disk_init(c3_c* pax_c, u3_disk_cb cb_u) if ( 0 == las_d ) { // fresh epoch (no events in lmdb yet) if ( 0 == lat_d ) { // first epoch log_u->dun_d = 0; - log_u->sen_d = 0; } else { // not first epoch log_u->dun_d = lat_d - 1; // set dun_d to last event in prev epoch - log_u->sen_d = log_u->dun_d; } } else { // not fresh epoch log_u->dun_d = las_d; // set dun_d to last event in lmdb - log_u->sen_d = las_d; } + log_u->sen_d = log_u->dun_d; + // mark the log as live log_u->liv_o = c3y; } @@ -992,6 +993,9 @@ c3_o u3_disk_epoc_init(u3_disk* log_u) { fclose(epv_f); // create binary version file + // XX make sure if you've updated your binary, a new epoch is created + // so that events within an epoch are guaranteed to have been processed + // by the binary version specified in the file c3_c biv_c[8193]; snprintf(biv_c, sizeof(biv_c), "%s/vere.txt", epo_c); FILE* biv_f = fopen(biv_c, "a"); @@ -1038,7 +1042,7 @@ c3_o u3_disk_epoc_init(u3_disk* log_u) { if ( c3y == eps_o ) { if ( c3n == u3_disk_save_meta(log_u->mdb_u, who_d, fak_o, lif_w) ) { fprintf(stderr, "disk: failed to save metadata\r\n"); - exit(1); + goto fail; } } @@ -1093,9 +1097,7 @@ c3_o u3_disk_epoc_last(u3_disk* log_u, c3_d* lat_d) { u3_dent* den_u = u3_foil_folder(log_u->com_u->pax_c)->dil_u; while ( den_u ) { c3_d epo_d = 0; - if ( 1 != sscanf(den_u->nam_c, "0i%" PRIu64, &epo_d) ) { - fprintf(stderr, "disk: epoch directory is not a @ui: %s\r\n", den_u->nam_c); - } else { + if ( 1 == sscanf(den_u->nam_c, "0i%" PRIu64, &epo_d) ) { ret_d = c3y; // return yes if at least one epoch directory exists } *lat_d = c3_max(epo_d, *lat_d); // update the latest epoch number @@ -1107,7 +1109,7 @@ c3_o u3_disk_epoc_last(u3_disk* log_u, c3_d* lat_d) { /* u3_disk_migrate: migrates disk format. */ -void u3_disk_migrate(u3_disk* log_u) +c3_o u3_disk_migrate(u3_disk* log_u) { // check if data.mdb exists in log directory c3_o dat_o = c3n; @@ -1128,12 +1130,9 @@ void u3_disk_migrate(u3_disk* log_u) // initialize first epoch "0i0" if ( c3n == u3_disk_epoc_init(log_u) ) { fprintf(stderr, "disk: migrate: failed to initialize first epoch\r\n"); - exit(1); + return c3n; } } else if ( c3n == epo_o ) { - // migrate existing, unversioned disk to v1 - fprintf(stderr, "disk: migrating disk to v%d format\r\n", U3D_VER1); - // initialize pre-migrated lmdb MDB_env* old_u; { @@ -1147,8 +1146,7 @@ void u3_disk_migrate(u3_disk* log_u) if ( 0 == (old_u = u3_lmdb_init(log_u->com_u->pax_c, siz_i)) ) { fprintf(stderr, "disk: failed to initialize database\r\n"); - c3_free(log_u); - return; + return c3n; } } @@ -1156,69 +1154,64 @@ void u3_disk_migrate(u3_disk* log_u) c3_d fir_d, las_d; if ( c3n == u3_lmdb_gulf(old_u, &fir_d, &las_d) ) { fprintf(stderr, "disk: failed to get first/last event numbers\r\n"); - return; + return c3n; } - // boot - u3m_boot(u3_Host.dir_c, (size_t)1 << u3_Host.ops_u.lom_y); - - // make sure there's a current snapshot - if ( las_d != u3A->eve_d ) { + // ensure there's a current snapshot without the below code + if ( u3_Host.eve_d != las_d ) { fprintf(stderr, "disk: migrate: error: snapshot is out of date, please " "start/shutdown your pier gracefully first\r\n"); - exit(1); + return c3n; } - // shutdown pre-migrated u3 system and lmdb - u3m_stop(); - // initialize first epoch "0i0" if ( c3n == u3_disk_epoc_init(log_u) ) { fprintf(stderr, "disk: migrate: failed to initialize first epoch\r\n"); - exit(1); + return c3n; } - // move data.mdb to 0i0/data.mdb - c3_c new_c[8193]; - snprintf(new_c, sizeof(new_c), "%s/0i0/data.mdb", log_u->com_u->pax_c); - if ( 0 != rename(dat_c, new_c) ) { + // move data.mdb and lock.mdb to 0i0/ + c3_c dut_c[8193], luk_c[8193]; + snprintf(dut_c, sizeof(dut_c), "%s/data.mdb", log_u->com_u->pax_c); + snprintf(luk_c, sizeof(luk_c), "%s/lock.mdb", log_u->com_u->pax_c); + + c3_c epo_c[8193], dat_c[8193], lok_c[8193]; + snprintf(epo_c, sizeof(epo_c), "%s/0i0", log_u->com_u->pax_c); + snprintf(dat_c, sizeof(dat_c), "%s/data.mdb", epo_c); + snprintf(lok_c, sizeof(lok_c), "%s/lock.mdb", epo_c); + + if ( 0 != c3_rename(dut_c, dat_c) ) { fprintf(stderr, "disk: migrate: failed to move data.mdb\r\n"); - exit(1); + goto _u3_disk_migrate_fail; } - - // move lock.mdb to 0i0/lock.mdb - c3_c old_c[8193]; - snprintf(old_c, sizeof(old_c), "%s/lock.mdb", log_u->com_u->pax_c); - snprintf(new_c, sizeof(new_c), "%s/0i0/lock.mdb", log_u->com_u->pax_c); - if ( 0 != rename(old_c, new_c) ) { + if ( 0 != c3_rename(luk_c, lok_c) ) { fprintf(stderr, "disk: migrate: failed to move lock.mdb\r\n"); - exit(1); + c3_rename(dat_c, dut_c); + goto _u3_disk_migrate_fail; } // rollover to new epoch log_u->mdb_u = old_u; if ( c3n == u3_disk_epoc_init(log_u) ) { fprintf(stderr, "roll: error: failed to initialize new epoch\r\n"); - exit(1); + goto _u3_disk_migrate_fail; } - // delete backup snapshot + // delete backup snapshot; migration still succeeds if this fails c3_c bhk_c[8193]; snprintf(bhk_c, sizeof(bhk_c), "%s/.urb/bhk", u3_Host.dir_c); - c3_c nop_c[8193]; + + c3_c nop_c[8193], sop_c[8193]; snprintf(nop_c, sizeof(nop_c), "%s/north.bin", bhk_c); - c3_c sop_c[8193]; snprintf(sop_c, sizeof(sop_c), "%s/south.bin", bhk_c); + if ( c3n == c3_unlink(nop_c) ) { - fprintf(stderr, "disk: migrate: failed to delete north.bin\r\n"); - exit(1); + fprintf(stderr, "disk: migrate: failed to delete bhk/north.bin\r\n"); } else if ( c3n == c3_unlink(sop_c) ) { - fprintf(stderr, "disk: migrate: failed to delete south.bin\r\n"); - exit(1); + fprintf(stderr, "disk: migrate: failed to delete bhk/south.bin\r\n"); } else { if ( c3n == c3_rmdir(bhk_c) ) { - fprintf(stderr, "disk: migrate: failed to delete bhk\r\n"); - exit(1); + fprintf(stderr, "disk: migrate: failed to delete bhk/\r\n"); } } @@ -1226,5 +1219,12 @@ void u3_disk_migrate(u3_disk* log_u) fprintf(stderr, "disk: migrated disk to v%d format\r\n", U3D_VER1); } - return; +_u3_disk_migrate_fail: +{ + c3_d lat_d; + u3_disk_epoc_last(log_u, &lat_d); + u3_disk_epoc_kill(log_u, lat_d); +} + + return c3y; } \ No newline at end of file diff --git a/pkg/vere/main.c b/pkg/vere/main.c index c4fd9de29c..3dbe724ea7 100644 --- a/pkg/vere/main.c +++ b/pkg/vere/main.c @@ -1971,6 +1971,8 @@ _cw_play(c3_i argc, c3_c* argv[]) static void _cw_prep(c3_i argc, c3_c* argv[]) { + // XX roll with old binary + // check that new epoch is empty, migrate snapshot in-place c3_i ch_i, lid_i; c3_w arg_w; @@ -2025,6 +2027,7 @@ _cw_prep(c3_i argc, c3_c* argv[]) static void _cw_chop(c3_i argc, c3_c* argv[]) { + // XX keep the last epoch (2 epochs left after chop) c3_i ch_i, lid_i; c3_w arg_w; @@ -2630,6 +2633,13 @@ main(c3_i argc, } } + // we need the current snapshot's latest event number to + // validate whether we can execute disk migration + if ( c3n == u3_Host.ops_u.nuu ) { + u3_Host.eve_d = u3m_boot(u3_Host.dir_c, (size_t)1 << u3_Host.ops_u.lom_y); + // XX u3e_close() + } + // starting u3m configures OpenSSL memory functions, so we must do it // before any OpenSSL allocations // diff --git a/pkg/vere/vere.h b/pkg/vere/vere.h index a2726d2341..eeaa8cec4f 100644 --- a/pkg/vere/vere.h +++ b/pkg/vere/vere.h @@ -312,6 +312,7 @@ typedef struct _u3_host { c3_w kno_w; // current executing stage c3_c* dir_c; // pier path (no trailing /) + c3_d eve_d; // initial current snapshot c3_c* dem_c; // daemon executable path c3_c* wrk_c; // worker executable path c3_d now_d; // event tick @@ -999,7 +1000,7 @@ /* u3_disk_migrate(): migrates disk format. */ - void + c3_o u3_disk_migrate(u3_disk* log_u); /* u3_lord_init(): start serf. From b93508e1704090ad2796b2e3c8a4071ab68b5fc8 Mon Sep 17 00:00:00 2001 From: Matthew LeVan Date: Fri, 28 Apr 2023 11:29:32 -0400 Subject: [PATCH 021/271] lmdb: factor siz_i calculation --- pkg/vere/db/lmdb.c | 11 ++++++++++- pkg/vere/db/lmdb.h | 2 +- pkg/vere/disk.c | 32 ++++---------------------------- 3 files changed, 15 insertions(+), 30 deletions(-) diff --git a/pkg/vere/db/lmdb.c b/pkg/vere/db/lmdb.c index 8f8864b77f..d9fed4c0de 100644 --- a/pkg/vere/db/lmdb.c +++ b/pkg/vere/db/lmdb.c @@ -38,11 +38,20 @@ intmax_t mdb_get_filesize(mdb_filehandle_t han_u); /* u3_lmdb_init(): open lmdb at [pax_c], mmap up to [siz_i]. */ MDB_env* -u3_lmdb_init(const c3_c* pax_c, size_t siz_i) +u3_lmdb_init(const c3_c* pax_c) { MDB_env* env_u; c3_w ret_w; + // calculate db size based on architecture + const size_t siz_i = + // 500 GiB is as large as musl on aarch64 wants to allow + #if (defined(U3_CPU_aarch64) && defined(U3_OS_linux)) + 0x7d00000000; + #else + 0x10000000000; + #endif + if ( (ret_w = mdb_env_create(&env_u)) ) { mdb_logerror(stderr, ret_w, "lmdb: init fail"); return 0; diff --git a/pkg/vere/db/lmdb.h b/pkg/vere/db/lmdb.h index aab4e6c9f7..7aa996001f 100644 --- a/pkg/vere/db/lmdb.h +++ b/pkg/vere/db/lmdb.h @@ -22,7 +22,7 @@ /* u3_lmdb_init(): open lmdb at [pax_c], mmap up to [siz_i]. */ MDB_env* - u3_lmdb_init(const c3_c* pax_c, size_t siz_i); + u3_lmdb_init(const c3_c* pax_c); /* u3_lmdb_exit(): close lmdb. */ diff --git a/pkg/vere/disk.c b/pkg/vere/disk.c index bd6f8a4d27..19dce9087c 100644 --- a/pkg/vere/disk.c +++ b/pkg/vere/disk.c @@ -906,16 +906,8 @@ u3_disk_init(c3_c* pax_c, u3_disk_cb cb_u) c3_c epo_c[8193]; snprintf(epo_c, 8192, "%s/0i%" PRIu64, log_c, lat_d); { - // XX extract into function? - const size_t siz_i = - // 500 GiB is as large as musl on aarch64 wants to allow - #if (defined(U3_CPU_aarch64) && defined(U3_OS_linux)) - 0x7d00000000; - #else - 0x10000000000; - #endif - - if ( 0 == (log_u->mdb_u = u3_lmdb_init(epo_c, siz_i)) ) { + + if ( 0 == (log_u->mdb_u = u3_lmdb_init(epo_c)) ) { fprintf(stderr, "disk: failed to initialize database\r\n"); c3_free(log_u); return 0; @@ -1023,15 +1015,7 @@ c3_o u3_disk_epoc_init(u3_disk* log_u) { // initialize db of new epoch { - const size_t siz_i = - // 500 GiB is as large as musl on aarch64 wants to allow - #if (defined(U3_CPU_aarch64) && defined(U3_OS_linux)) - 0x7d00000000; - #else - 0x10000000000; - #endif - - if ( 0 == (log_u->mdb_u = u3_lmdb_init(epo_c, siz_i)) ) { + if ( 0 == (log_u->mdb_u = u3_lmdb_init(epo_c)) ) { fprintf(stderr, "disk: failed to initialize database\r\n"); c3_free(log_u); goto fail; @@ -1136,15 +1120,7 @@ c3_o u3_disk_migrate(u3_disk* log_u) // initialize pre-migrated lmdb MDB_env* old_u; { - const size_t siz_i = - // 500 GiB is as large as musl on aarch64 wants to allow - #if (defined(U3_CPU_aarch64) && defined(U3_OS_linux)) - 0x7d00000000; - #else - 0x10000000000; - #endif - - if ( 0 == (old_u = u3_lmdb_init(log_u->com_u->pax_c, siz_i)) ) { + if ( 0 == (old_u = u3_lmdb_init(log_u->com_u->pax_c)) ) { fprintf(stderr, "disk: failed to initialize database\r\n"); return c3n; } From bb1f786c2a72abe6e111262e0b8c435d79defa2c Mon Sep 17 00:00:00 2001 From: Matthew LeVan Date: Fri, 28 Apr 2023 13:04:46 -0400 Subject: [PATCH 022/271] disk: create new epoch when vere version is different --- pkg/vere/disk.c | 69 +++++++++++++++++++++++++++++++++++++------------ pkg/vere/main.c | 24 ++++++++++++++--- 2 files changed, 74 insertions(+), 19 deletions(-) diff --git a/pkg/vere/disk.c b/pkg/vere/disk.c index 19dce9087c..2c1d1e0622 100644 --- a/pkg/vere/disk.c +++ b/pkg/vere/disk.c @@ -902,9 +902,48 @@ u3_disk_init(c3_c* pax_c, u3_disk_cb cb_u) return 0; } - // initialize epoch's db + // set path to latest epoch c3_c epo_c[8193]; snprintf(epo_c, 8192, "%s/0i%" PRIu64, log_c, lat_d); + + // get vere version from the latest epoch + c3_c fil_c[8193]; + snprintf(fil_c, 8192, "%s/vere.txt", epo_c); + if ( 0 != access(fil_c, F_OK) ) { + fprintf(stderr, "disk: failed to access %s\r\n", fil_c); + c3_free(log_u); + return 0; + } + + // read string from vere.txt file + c3_c ver_c[8193]; + { + FILE* fil_u = fopen(fil_c, "r"); + if ( !fil_u ) { + fprintf(stderr, "disk: failed to open %s\r\n", fil_c); + c3_free(log_u); + return 0; + } + if ( !fgets(ver_c, 8192, fil_u) ) { + fprintf(stderr, "disk: failed to read %s\r\n", fil_c); + c3_free(log_u); + return 0; + } + fclose(fil_u); + } + + // if binary version from vere.txt is different + // than current version, then create a new epoch + if ( 0 != strcmp(ver_c, URBIT_VERSION) ) { + fprintf(stderr, "disk: binary version mismatch, creating new epoch\r\n"); + if ( c3n == u3_disk_epoc_init(log_u) ) { + fprintf(stderr, "disk: failed to initialize epoch\r\n"); + c3_free(log_u); + return 0; + } + } + + // initialize epoch's db { if ( 0 == (log_u->mdb_u = u3_lmdb_init(epo_c)) ) { @@ -956,16 +995,12 @@ c3_o u3_disk_epoc_init(u3_disk* log_u) { if ( c3n == eps_o ) { new_d = 0; // no epochs yet, so create first one 0i0 } else { - // ensure latest epoch isn't empty + // get first/last event numbers c3_d fir_d, las_d; if ( c3n == u3_lmdb_gulf(log_u->mdb_u, &fir_d, &las_d) ) { fprintf(stderr, "disk: failed to get first/last event numbers\r\n"); return c3n; } - if ( fir_d == las_d ) { - fprintf(stderr, "disk: latest epoch is empty; skipping rollover\r\n"); - return c3n; - } new_d = 1 + las_d; // create next epoch } @@ -1117,6 +1152,9 @@ c3_o u3_disk_migrate(u3_disk* log_u) return c3n; } } else if ( c3n == epo_o ) { + // migrate existing pier + fprintf(stderr, "disk: migrating disk to v%d format\r\n", U3D_VER1); + // initialize pre-migrated lmdb MDB_env* old_u; { @@ -1158,19 +1196,25 @@ c3_o u3_disk_migrate(u3_disk* log_u) if ( 0 != c3_rename(dut_c, dat_c) ) { fprintf(stderr, "disk: migrate: failed to move data.mdb\r\n"); - goto _u3_disk_migrate_fail; + u3_disk_epoc_last(log_u, &lat_d); + u3_disk_epoc_kill(log_u, lat_d); + return c3n; } if ( 0 != c3_rename(luk_c, lok_c) ) { fprintf(stderr, "disk: migrate: failed to move lock.mdb\r\n"); c3_rename(dat_c, dut_c); - goto _u3_disk_migrate_fail; + u3_disk_epoc_last(log_u, &lat_d); + u3_disk_epoc_kill(log_u, lat_d); + return c3n; } // rollover to new epoch log_u->mdb_u = old_u; if ( c3n == u3_disk_epoc_init(log_u) ) { fprintf(stderr, "roll: error: failed to initialize new epoch\r\n"); - goto _u3_disk_migrate_fail; + u3_disk_epoc_last(log_u, &lat_d); + u3_disk_epoc_kill(log_u, lat_d); + return c3n; } // delete backup snapshot; migration still succeeds if this fails @@ -1195,12 +1239,5 @@ c3_o u3_disk_migrate(u3_disk* log_u) fprintf(stderr, "disk: migrated disk to v%d format\r\n", U3D_VER1); } -_u3_disk_migrate_fail: -{ - c3_d lat_d; - u3_disk_epoc_last(log_u, &lat_d); - u3_disk_epoc_kill(log_u, lat_d); -} - return c3y; } \ No newline at end of file diff --git a/pkg/vere/main.c b/pkg/vere/main.c index 3dbe724ea7..ba073d1b21 100644 --- a/pkg/vere/main.c +++ b/pkg/vere/main.c @@ -2088,7 +2088,16 @@ _cw_chop(c3_i argc, c3_c* argv[]) } // create new epoch - if ( c3n == u3_disk_epoc_init(log_u) ) { + c3_d fir_d, las_d; + if ( c3n == u3_lmdb_gulf(log_u->mdb_u, &fir_d, &las_d) ) { + fprintf(stderr, "chop: failed to get first/last events\r\n"); + exit(1); + } + if ( fir_d == las_d == 0 ) { + fprintf(stderr, "chop: latest epoch already empty\r\n"); + exit(1); + } + else if ( c3n == u3_disk_epoc_init(log_u) ) { fprintf(stderr, "chop: failed to create new epoch\r\n"); exit(1); } @@ -2189,8 +2198,17 @@ _cw_roll(c3_i argc, c3_c* argv[]) } // create new epoch - if ( c3n == u3_disk_epoc_init(log_u) ) { - fprintf(stderr, "roll: error: failed to initialize new epoch\r\n"); + c3_d fir_d, las_d; + if ( c3n == u3_lmdb_gulf(log_u->mdb_u, &fir_d, &las_d) ) { + fprintf(stderr, "roll: failed to get first/last events\r\n"); + exit(1); + } + if ( fir_d == las_d == 0 ) { + fprintf(stderr, "roll: latest epoch already empty\r\n"); + exit(1); + } + else if ( c3n == u3_disk_epoc_init(log_u) ) { + fprintf(stderr, "roll: failed to create new epoch\r\n"); exit(1); } From 5b432b6b3ec8188d002ff8a32f0c52522dc0c1e9 Mon Sep 17 00:00:00 2001 From: Matthew LeVan Date: Mon, 1 May 2023 08:54:08 -0400 Subject: [PATCH 023/271] cli: make `chop` leave only the latest two epochs --- pkg/vere/disk.c | 42 ++---------------------------------------- pkg/vere/main.c | 25 ++++++++++++++++--------- 2 files changed, 18 insertions(+), 49 deletions(-) diff --git a/pkg/vere/disk.c b/pkg/vere/disk.c index 2c1d1e0622..d4ca5a21b8 100644 --- a/pkg/vere/disk.c +++ b/pkg/vere/disk.c @@ -906,43 +906,6 @@ u3_disk_init(c3_c* pax_c, u3_disk_cb cb_u) c3_c epo_c[8193]; snprintf(epo_c, 8192, "%s/0i%" PRIu64, log_c, lat_d); - // get vere version from the latest epoch - c3_c fil_c[8193]; - snprintf(fil_c, 8192, "%s/vere.txt", epo_c); - if ( 0 != access(fil_c, F_OK) ) { - fprintf(stderr, "disk: failed to access %s\r\n", fil_c); - c3_free(log_u); - return 0; - } - - // read string from vere.txt file - c3_c ver_c[8193]; - { - FILE* fil_u = fopen(fil_c, "r"); - if ( !fil_u ) { - fprintf(stderr, "disk: failed to open %s\r\n", fil_c); - c3_free(log_u); - return 0; - } - if ( !fgets(ver_c, 8192, fil_u) ) { - fprintf(stderr, "disk: failed to read %s\r\n", fil_c); - c3_free(log_u); - return 0; - } - fclose(fil_u); - } - - // if binary version from vere.txt is different - // than current version, then create a new epoch - if ( 0 != strcmp(ver_c, URBIT_VERSION) ) { - fprintf(stderr, "disk: binary version mismatch, creating new epoch\r\n"); - if ( c3n == u3_disk_epoc_init(log_u) ) { - fprintf(stderr, "disk: failed to initialize epoch\r\n"); - c3_free(log_u); - return 0; - } - } - // initialize epoch's db { @@ -966,12 +929,11 @@ u3_disk_init(c3_c* pax_c, u3_disk_cb cb_u) if ( 0 == lat_d ) { // first epoch log_u->dun_d = 0; } else { // not first epoch - log_u->dun_d = lat_d - 1; // set dun_d to last event in prev epoch + log_u->dun_d = lat_d; // set dun_d to last event in prev epoch } } else { // not fresh epoch log_u->dun_d = las_d; // set dun_d to last event in lmdb } - log_u->sen_d = log_u->dun_d; // mark the log as live @@ -1001,7 +963,7 @@ c3_o u3_disk_epoc_init(u3_disk* log_u) { fprintf(stderr, "disk: failed to get first/last event numbers\r\n"); return c3n; } - new_d = 1 + las_d; // create next epoch + new_d = las_d; // create next epoch } // create new epoch directory diff --git a/pkg/vere/main.c b/pkg/vere/main.c index ba073d1b21..b122aa45ab 100644 --- a/pkg/vere/main.c +++ b/pkg/vere/main.c @@ -2027,7 +2027,6 @@ _cw_prep(c3_i argc, c3_c* argv[]) static void _cw_chop(c3_i argc, c3_c* argv[]) { - // XX keep the last epoch (2 epochs left after chop) c3_i ch_i, lid_i; c3_w arg_w; @@ -2087,13 +2086,20 @@ _cw_chop(c3_i argc, c3_c* argv[]) exit(1); } + // get latest epoch number prior to creating a new one + c3_d pre_d; + if ( c3n == u3_disk_epoc_last(log_u, &pre_d) ) { + fprintf(stderr, "chop: failed to find last epoch\r\n"); + exit(1); + } + // create new epoch c3_d fir_d, las_d; if ( c3n == u3_lmdb_gulf(log_u->mdb_u, &fir_d, &las_d) ) { fprintf(stderr, "chop: failed to get first/last events\r\n"); exit(1); } - if ( fir_d == las_d == 0 ) { + if ( fir_d == las_d ) { fprintf(stderr, "chop: latest epoch already empty\r\n"); exit(1); } @@ -2102,20 +2108,20 @@ _cw_chop(c3_i argc, c3_c* argv[]) exit(1); } - // get latest epoch number - c3_d lat_d; - if ( c3n == u3_disk_epoc_last(log_u, &lat_d) ) { + // get latest epoch number prior to creating a new one + c3_d pos_d; + if ( c3n == u3_disk_epoc_last(log_u, &pos_d) ) { fprintf(stderr, "chop: failed to find last epoch\r\n"); exit(1); } - // delete all but the newly created epoch + // delete all but the last two epochs u3_dent* den_u = log_u->com_u->dil_u; c3_d epo_d = 0; - while ( den_u && epo_d <= lat_d ) { + while ( den_u ) { if ( 1 != sscanf(den_u->nam_c, "0i%" PRIu64, &epo_d) ) { fprintf(stderr, "disk: epoch directory is not a @ui: %s\r\n", den_u->nam_c); - } else { + } else if ( epo_d != pre_d && epo_d != pos_d ) { fprintf(stderr, "chop: deleting epoch 0i%" PRIu64 "\r\n", epo_d); if ( c3y != u3_disk_epoc_kill(log_u, epo_d) ) { fprintf(stderr, "chop: failed to delete epoch 0i%" PRIu64 "\r\n", epo_d); @@ -2203,7 +2209,8 @@ _cw_roll(c3_i argc, c3_c* argv[]) fprintf(stderr, "roll: failed to get first/last events\r\n"); exit(1); } - if ( fir_d == las_d == 0 ) { + + if ( fir_d == las_d ) { fprintf(stderr, "roll: latest epoch already empty\r\n"); exit(1); } From 2e9da020793173c32d3589330f7b148f11530f1c Mon Sep 17 00:00:00 2001 From: Matthew LeVan Date: Mon, 1 May 2023 10:16:10 -0400 Subject: [PATCH 024/271] disk: create a new epoch when version change detected --- pkg/vere/disk.c | 56 ++++++++++++++++++++++++++++++++++++++++--------- pkg/vere/main.c | 5 ++--- pkg/vere/vere.h | 5 +++++ 3 files changed, 53 insertions(+), 13 deletions(-) diff --git a/pkg/vere/disk.c b/pkg/vere/disk.c index d4ca5a21b8..866e6b9391 100644 --- a/pkg/vere/disk.c +++ b/pkg/vere/disk.c @@ -875,7 +875,6 @@ u3_disk_init(c3_c* pax_c, u3_disk_cb cb_u) } // create/load $pier/.urb/log - // XX closures? // { c3_c log_c[8193]; @@ -902,18 +901,23 @@ u3_disk_init(c3_c* pax_c, u3_disk_cb cb_u) return 0; } + // get binary version from latest epoch + c3_c ver_w[8193]; + if ( c3n == u3_disk_epoc_vere(log_u, lat_d, ver_w) ) { + fprintf(stderr, "disk: failed to load epoch version\r\n"); + c3_free(log_u); + return 0; + } + // set path to latest epoch c3_c epo_c[8193]; snprintf(epo_c, 8192, "%s/0i%" PRIu64, log_c, lat_d); - // initialize epoch's db - { - - if ( 0 == (log_u->mdb_u = u3_lmdb_init(epo_c)) ) { - fprintf(stderr, "disk: failed to initialize database\r\n"); - c3_free(log_u); - return 0; - } + // initialize latest epoch's db + if ( 0 == (log_u->mdb_u = u3_lmdb_init(epo_c)) ) { + fprintf(stderr, "disk: failed to initialize database\r\n"); + c3_free(log_u); + return 0; } fprintf(stderr, "disk: loaded epoch 0i%" PRIu64 "\r\n", lat_d); @@ -936,6 +940,16 @@ u3_disk_init(c3_c* pax_c, u3_disk_cb cb_u) } log_u->sen_d = log_u->dun_d; + // if binary version of latest epoch is not the same as the + // running binary, then we need to create a new epoch + if ( 0 != strcmp(ver_w, URBIT_VERSION) ) { + if ( c3n == u3_disk_epoc_init(log_u) ) { + fprintf(stderr, "disk: failed to initialize epoch\r\n"); + c3_free(log_u); + return 0; + } + } + // mark the log as live log_u->liv_o = c3y; } @@ -1088,6 +1102,29 @@ c3_o u3_disk_epoc_last(u3_disk* log_u, c3_d* lat_d) { return ret_d; } +/* u3_disk_epoc_vere: get binary version from epoch. +*/ +c3_o +u3_disk_epoc_vere(u3_disk* log_u, c3_d epo_d, c3_c* ver_w) { + c3_c ver_c[8193]; + snprintf(ver_c, sizeof(ver_c), "%s/0i%" PRIu64 "/vere.txt", + log_u->com_u->pax_c, epo_d); + + FILE* fil_u = fopen(ver_c, "r"); + if ( NULL == fil_u ) { + fprintf(stderr, "disk: failed to open vere.txt in epoch 0i%" PRIu64 + "\r\n", epo_d); + return c3n; + } + + if ( 1 != fscanf(fil_u, "%s", ver_w) ) { + fprintf(stderr, "disk: failed to read vere.txt in epoch 0i%" PRIu64 + "\r\n", epo_d); + return c3n; + } + return c3y; +} + /* u3_disk_migrate: migrates disk format. */ c3_o u3_disk_migrate(u3_disk* log_u) @@ -1115,7 +1152,6 @@ c3_o u3_disk_migrate(u3_disk* log_u) } } else if ( c3n == epo_o ) { // migrate existing pier - fprintf(stderr, "disk: migrating disk to v%d format\r\n", U3D_VER1); // initialize pre-migrated lmdb MDB_env* old_u; diff --git a/pkg/vere/main.c b/pkg/vere/main.c index b122aa45ab..838cac5388 100644 --- a/pkg/vere/main.c +++ b/pkg/vere/main.c @@ -2119,9 +2119,8 @@ _cw_chop(c3_i argc, c3_c* argv[]) u3_dent* den_u = log_u->com_u->dil_u; c3_d epo_d = 0; while ( den_u ) { - if ( 1 != sscanf(den_u->nam_c, "0i%" PRIu64, &epo_d) ) { - fprintf(stderr, "disk: epoch directory is not a @ui: %s\r\n", den_u->nam_c); - } else if ( epo_d != pre_d && epo_d != pos_d ) { + c3_d res_d = sscanf(den_u->nam_c, "0i%" PRIu64, &epo_d); + if ( (1 == res_d) && (epo_d != pre_d) && (epo_d != pos_d) ) { fprintf(stderr, "chop: deleting epoch 0i%" PRIu64 "\r\n", epo_d); if ( c3y != u3_disk_epoc_kill(log_u, epo_d) ) { fprintf(stderr, "chop: failed to delete epoch 0i%" PRIu64 "\r\n", epo_d); diff --git a/pkg/vere/vere.h b/pkg/vere/vere.h index eeaa8cec4f..f54fa2a6d6 100644 --- a/pkg/vere/vere.h +++ b/pkg/vere/vere.h @@ -998,6 +998,11 @@ c3_o u3_disk_epoc_last(u3_disk* log_u, c3_d* lat_d); + /* u3_disk_epoc_vere(): get binary version from epoch. + */ + c3_o + u3_disk_epoc_vere(u3_disk* log_u, c3_d epo_d, c3_c* ver_w); + /* u3_disk_migrate(): migrates disk format. */ c3_o From 1b952d06aecc24dde76f9944466c71149fd905d2 Mon Sep 17 00:00:00 2001 From: Matthew LeVan Date: Tue, 16 May 2023 12:02:01 -0400 Subject: [PATCH 025/271] c3: add `c3_link` --- pkg/c3/defs.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/c3/defs.h b/pkg/c3/defs.h index 38b39263a9..bbf8d91ef7 100644 --- a/pkg/c3/defs.h +++ b/pkg/c3/defs.h @@ -149,6 +149,8 @@ mkdir(a, b);}) # define c3_rmdir(a) ({ \ rmdir(a);}) +# define c3_link(a, b) ({ \ + link(a, b);}) # define c3_unlink(a) ({ \ unlink(a);}) # define c3_fopen(a, b) ({ \ From b0da9beb58ee47ac306fe03d01e8ac461156ce13 Mon Sep 17 00:00:00 2001 From: Matthew LeVan Date: Thu, 18 May 2023 13:37:08 -0400 Subject: [PATCH 026/271] disk: make migration idempotent (crash safe) --- pkg/vere/disk.c | 212 +++++++++++++++++++++++++++++++++++++----------- pkg/vere/main.c | 4 +- pkg/vere/vere.h | 5 ++ 3 files changed, 173 insertions(+), 48 deletions(-) diff --git a/pkg/vere/disk.c b/pkg/vere/disk.c index 866e6b9391..3985528493 100644 --- a/pkg/vere/disk.c +++ b/pkg/vere/disk.c @@ -962,6 +962,87 @@ u3_disk_init(c3_c* pax_c, u3_disk_cb cb_u) return log_u; } +/* u3_disk_epoc_good: check for valid epoch. +*/ +c3_o u3_disk_epoc_good(u3_disk* log_u, c3_d epo_d) { + /* a "valid" epoch is currently defined as a writable folder in + * the /.urb/log directory named with a @ui and contains: + * - a writable data.mdb file which can be opened by lmdb (add db checking?) + * - a writable epoc.txt file + * - a writable vere.txt file + * + * XX: should we check if the correct event sequence exists in the data.mdb file? + * + * note that this does not check if the epoch even contains snapshot files, + * let alone whether they're valid or not + */ + + c3_o ret_o = c3y; // return code + + // file paths + c3_c epo_c[8193], dat_c[8193], epv_c[8193], biv_c[8193]; + snprintf(epo_c, sizeof(epo_c), "%s/0i%" PRIu64, log_u->com_u->pax_c, epo_d); + snprintf(dat_c, sizeof(dat_c), "%s/data.mdb", epo_c); + snprintf(epv_c, sizeof(epv_c), "%s/epoc.txt", epo_c); + snprintf(biv_c, sizeof(biv_c), "%s/vere.txt", epo_c); + + c3_o dir_o = c3n; // directory is writable + c3_o dat_o = c3n; // data.mdb is writable + c3_o mdb_o = c3n; // data.mdb can be opened + c3_o epv_o = c3n; // epoc.txt is writable + c3_o biv_o = c3n; // vere.txt is writable + + // check if dir is writable + if ( 0 == access(epo_c, W_OK) ) { + dir_o = c3y; + } + + // check if data.mdb is writable + if ( 0 == access(dat_c, W_OK) ) { + dat_o = c3y; + // check if we can open data.mdb + MDB_env* env_u; + if ( 0 != (env_u = u3_lmdb_init(epo_c)) ) { + mdb_o = c3y; + } + u3_lmdb_exit(env_u); + } + + // check if epoc.txt is writable + if ( 0 == access(epv_c, W_OK) ) { + epv_o = c3y; + } + + // check if vere.txt is writable + if ( 0 == access(biv_c, W_OK) ) { + biv_o = c3y; + } + + // print error messages + if ( c3n == dir_o ) { + fprintf(stderr, "disk: epoch 0i%" PRIu64 " is not writable\r\n", epo_d); + ret_o = c3n; + } + if ( c3n == dat_o ) { + fprintf(stderr, "disk: epoch 0i%" PRIu64 "/data.mdb is not writable\r\n", epo_d); + ret_o = c3n; + } + if ( c3n == mdb_o ) { + fprintf(stderr, "disk: epoch 0i%" PRIu64 "/data.mdb can't be opened\r\n", epo_d); + ret_o = c3n; + } + if ( c3n == epv_o ) { + fprintf(stderr, "disk: epoch 0i%" PRIu64 "/epoc.txt is not writable\r\n", epo_d); + ret_o = c3n; + } + if ( c3n == biv_o ) { + fprintf(stderr, "disk: epoch 0i%" PRIu64 "/vere.txt is not writable\r\n", epo_d); + ret_o = c3n; + } + + return ret_o; +} + /* u3_disk_epoc_init: create new epoch. */ c3_o u3_disk_epoc_init(u3_disk* log_u) { @@ -969,8 +1050,14 @@ c3_o u3_disk_epoc_init(u3_disk* log_u) { c3_d lat_d, new_d; c3_o eps_o = u3_disk_epoc_last(log_u, &lat_d); if ( c3n == eps_o ) { - new_d = 0; // no epochs yet, so create first one 0i0 + // no epochs yet, so create first one 0i0 + new_d = 0; + } else if ( c3n == u3_disk_epoc_good(log_u, lat_d) ) { + // last epoch is invalid, so overwrite it + new_d = lat_d; } else { + // last epoch is valid, so create next one + // get first/last event numbers c3_d fir_d, las_d; if ( c3n == u3_lmdb_gulf(log_u->mdb_u, &fir_d, &las_d) ) { @@ -980,28 +1067,26 @@ c3_o u3_disk_epoc_init(u3_disk* log_u) { new_d = las_d; // create next epoch } - // create new epoch directory + // create new epoch directory if it doesn't exist c3_c epo_c[8193]; snprintf(epo_c, sizeof(epo_c), "%s/0i%" PRIu64, log_u->com_u->pax_c, new_d); - if ( 0 != c3_mkdir(epo_c, 0700) ) { + c3_d ret_d = c3_mkdir(epo_c, 0700); + if ( ( ret_d < 0 ) && ( errno != EEXIST ) ) { fprintf(stderr, "disk: failed to create epoch directory %" PRIu64 "\r\n", new_d); return c3n; } - // create epoch version file + // create epoch version file, overwriting any existing file c3_c epv_c[8193]; snprintf(epv_c, sizeof(epv_c), "%s/epoc.txt", epo_c); - FILE* epv_f = fopen(epv_c, "a"); + FILE* epv_f = fopen(epv_c, "w"); fprintf(epv_f, "%d\n", U3D_VER1); fclose(epv_f); - // create binary version file - // XX make sure if you've updated your binary, a new epoch is created - // so that events within an epoch are guaranteed to have been processed - // by the binary version specified in the file + // create binary version file, overwriting any existing file c3_c biv_c[8193]; snprintf(biv_c, sizeof(biv_c), "%s/vere.txt", epo_c); - FILE* biv_f = fopen(biv_c, "a"); + FILE* biv_f = fopen(biv_c, "w"); fprintf(biv_f, URBIT_VERSION); fclose(biv_f); @@ -1013,7 +1098,7 @@ c3_o u3_disk_epoc_init(u3_disk* log_u) { } } - // get metadata from old epoch's db + // get metadata from old epoch or unmigrated event log's db c3_d who_d[2]; c3_o fak_o; c3_w lif_w; @@ -1025,7 +1110,13 @@ c3_o u3_disk_epoc_init(u3_disk* log_u) { } // initialize db of new epoch - { + if ( c3y == u3_Host.ops_u.nuu || new_d > 0 ) { + c3_c dat_c[8193]; + snprintf(dat_c, sizeof(dat_c), "%s/data.mdb", epo_c); + // if ( c3n == c3_unlink(dat_c) ) { + // fprintf(stderr, "disk: failed to rm 0i%" PRIu64 "/data.mdb\r\n", new_d); + // goto fail; + // }; if ( 0 == (log_u->mdb_u = u3_lmdb_init(epo_c)) ) { fprintf(stderr, "disk: failed to initialize database\r\n"); c3_free(log_u); @@ -1093,7 +1184,7 @@ c3_o u3_disk_epoc_last(u3_disk* log_u, c3_d* lat_d) { while ( den_u ) { c3_d epo_d = 0; if ( 1 == sscanf(den_u->nam_c, "0i%" PRIu64, &epo_d) ) { - ret_d = c3y; // return yes if at least one epoch directory exists + ret_d = c3y; // NB: returns yes if the directory merely exists } *lat_d = c3_max(epo_d, *lat_d); // update the latest epoch number den_u = den_u->nex_u; @@ -1129,19 +1220,42 @@ u3_disk_epoc_vere(u3_disk* log_u, c3_d epo_d, c3_c* ver_w) { */ c3_o u3_disk_migrate(u3_disk* log_u) { - // check if data.mdb exists in log directory + /* migration steps: + * 0. detect whether we need to migrate or not + * a. if it's a fresh boot via u3_Host.ops_u.nuu -> skip migration + * b. if data.mdb is readable in log directory -> execute migration + * 1. initialize epoch 0i0 (first call to u3_disk_epoc_init()) + * a. creates epoch directory + * b. creates epoch version file + * c. creates binary version file + * d. initializes database + * e. reads metadata from old database + * f. writes metadata to new database + * g. loads new epoch directory and sets it in log_u + * 2. create hard links to data.mdb and lock.mdb in 0i0/ + * 3. rollover to new epoch (second call to u3_disk_epoc_init()) + * a. same as 1a-g but also copies current snapshot between c/d steps + * 4. delete backup snapshot (c3_unlink() and c3_rmdir() calls) + * 5. delete old data.mdb and lock.mdb files (c3_unlink() calls) + */ + + // check if data.mdb is readable in log directory c3_o dat_o = c3n; c3_c dat_c[8193]; snprintf(dat_c, sizeof(dat_c), "%s/data.mdb", log_u->com_u->pax_c); - if ( 0 == access(dat_c, F_OK) ) { + if ( 0 == access(dat_c, R_OK) ) { dat_o = c3y; } - // check if any epochs exist - c3_d lat_d; - c3_o epo_o = u3_disk_epoc_last(log_u, &lat_d); + // check if lock.mdb is readable in log directory + c3_o lok_o = c3n; + c3_c lok_c[8193]; + snprintf(lok_c, sizeof(dat_c), "%s/data.mdb", log_u->com_u->pax_c); + if ( 0 == access(dat_c, R_OK) ) { + lok_o = c3y; + } - if ( c3n == epo_o && c3n == dat_o ) { + if ( c3y == u3_Host.ops_u.nuu ) { // initialize disk v1 on fresh boots fprintf(stderr, "disk: initializing disk with v%d format\r\n", U3D_VER1); @@ -1150,13 +1264,14 @@ c3_o u3_disk_migrate(u3_disk* log_u) fprintf(stderr, "disk: migrate: failed to initialize first epoch\r\n"); return c3n; } - } else if ( c3n == epo_o ) { - // migrate existing pier + } else if ( c3y == dat_o ) { + // migrate existing pier which has either: + // - not started the migration, or + // - crashed before completing the migration // initialize pre-migrated lmdb - MDB_env* old_u; { - if ( 0 == (old_u = u3_lmdb_init(log_u->com_u->pax_c)) ) { + if ( 0 == (log_u->mdb_u = u3_lmdb_init(log_u->com_u->pax_c)) ) { fprintf(stderr, "disk: failed to initialize database\r\n"); return c3n; } @@ -1164,15 +1279,17 @@ c3_o u3_disk_migrate(u3_disk* log_u) // get first/last event numbers from pre-migrated lmdb c3_d fir_d, las_d; - if ( c3n == u3_lmdb_gulf(old_u, &fir_d, &las_d) ) { + if ( c3n == u3_lmdb_gulf(log_u->mdb_u, &fir_d, &las_d) ) { fprintf(stderr, "disk: failed to get first/last event numbers\r\n"); return c3n; } - // ensure there's a current snapshot without the below code + // ensure there's a current snapshot if ( u3_Host.eve_d != las_d ) { fprintf(stderr, "disk: migrate: error: snapshot is out of date, please " "start/shutdown your pier gracefully first\r\n"); + fprintf(stderr, "disk: migrate: eve_d (%" PRIu64 ") != las_d (%" PRIu64 ")\r\n", + u3_Host.eve_d, las_d); return c3n; } @@ -1182,47 +1299,41 @@ c3_o u3_disk_migrate(u3_disk* log_u) return c3n; } - // move data.mdb and lock.mdb to 0i0/ - c3_c dut_c[8193], luk_c[8193]; + // create hard links to data.mdb and lock.mdb in 0i0/ + c3_c dut_c[8193], luk_c[8193]; // old paths snprintf(dut_c, sizeof(dut_c), "%s/data.mdb", log_u->com_u->pax_c); snprintf(luk_c, sizeof(luk_c), "%s/lock.mdb", log_u->com_u->pax_c); - c3_c epo_c[8193], dat_c[8193], lok_c[8193]; + c3_c epo_c[8193], dat_c[8193], lok_c[8193]; // new paths snprintf(epo_c, sizeof(epo_c), "%s/0i0", log_u->com_u->pax_c); snprintf(dat_c, sizeof(dat_c), "%s/data.mdb", epo_c); snprintf(lok_c, sizeof(lok_c), "%s/lock.mdb", epo_c); - if ( 0 != c3_rename(dut_c, dat_c) ) { - fprintf(stderr, "disk: migrate: failed to move data.mdb\r\n"); - u3_disk_epoc_last(log_u, &lat_d); - u3_disk_epoc_kill(log_u, lat_d); + if ( 0 < c3_link(dut_c, dat_c) ) { + fprintf(stderr, "disk: migrate: failed to create data.mdb hard link\r\n"); + fprintf(stderr, "errno %d: %s\r\n", errno, strerror(errno)); return c3n; } - if ( 0 != c3_rename(luk_c, lok_c) ) { - fprintf(stderr, "disk: migrate: failed to move lock.mdb\r\n"); - c3_rename(dat_c, dut_c); - u3_disk_epoc_last(log_u, &lat_d); - u3_disk_epoc_kill(log_u, lat_d); - return c3n; + if ( c3y == lok_o ) { // only link lock.mdb if it exists + if ( 0 < c3_link(luk_c, lok_c) ) { + fprintf(stderr, "disk: migrate: failed to create lock.mdb hard link\r\n"); + c3_rename(dat_c, dut_c); + return c3n; + } } // rollover to new epoch - log_u->mdb_u = old_u; + // log_u->mdb_u = old_u; if ( c3n == u3_disk_epoc_init(log_u) ) { fprintf(stderr, "roll: error: failed to initialize new epoch\r\n"); - u3_disk_epoc_last(log_u, &lat_d); - u3_disk_epoc_kill(log_u, lat_d); return c3n; } - // delete backup snapshot; migration still succeeds if this fails - c3_c bhk_c[8193]; + // delete backup snapshot + c3_c bhk_c[8193], nop_c[8193], sop_c[8193]; snprintf(bhk_c, sizeof(bhk_c), "%s/.urb/bhk", u3_Host.dir_c); - - c3_c nop_c[8193], sop_c[8193]; snprintf(nop_c, sizeof(nop_c), "%s/north.bin", bhk_c); snprintf(sop_c, sizeof(sop_c), "%s/south.bin", bhk_c); - if ( c3n == c3_unlink(nop_c) ) { fprintf(stderr, "disk: migrate: failed to delete bhk/north.bin\r\n"); } else if ( c3n == c3_unlink(sop_c) ) { @@ -1233,6 +1344,15 @@ c3_o u3_disk_migrate(u3_disk* log_u) } } + // delete old lock.mdb and data.mdb files + if ( 0 != c3_unlink(luk_c) ) { + fprintf(stderr, "disk: migrate: failed to unlink lock.mdb\r\n"); + } + if ( 0 != c3_unlink(dut_c) ) { + fprintf(stderr, "disk: migrate: failed to unlink data.mdb\r\n"); + return c3n; // migration succeeds only if we can unlink data.mdb + } + // success fprintf(stderr, "disk: migrated disk to v%d format\r\n", U3D_VER1); } diff --git a/pkg/vere/main.c b/pkg/vere/main.c index 838cac5388..7d69ed179f 100644 --- a/pkg/vere/main.c +++ b/pkg/vere/main.c @@ -2220,7 +2220,7 @@ _cw_roll(c3_i argc, c3_c* argv[]) // success c3_d epo_d = log_u->dun_d + 1; - fprintf(stderr, "roll: epoch rollover complete"); + fprintf(stderr, "roll: epoch rollover complete\r\n"); } /* _cw_vere(): download vere @@ -2659,7 +2659,7 @@ main(c3_i argc, // we need the current snapshot's latest event number to // validate whether we can execute disk migration - if ( c3n == u3_Host.ops_u.nuu ) { + if ( u3_Host.ops_u.nuu == c3n ) { u3_Host.eve_d = u3m_boot(u3_Host.dir_c, (size_t)1 << u3_Host.ops_u.lom_y); // XX u3e_close() } diff --git a/pkg/vere/vere.h b/pkg/vere/vere.h index f54fa2a6d6..1e76053d10 100644 --- a/pkg/vere/vere.h +++ b/pkg/vere/vere.h @@ -982,6 +982,11 @@ */ void u3_disk_plan(u3_disk* log_u, u3_fact* tac_u); + + /* u3_disk_epoc_good(): check for valid epoch. + */ + c3_o + u3_disk_epoc_good(u3_disk* log_u, c3_d epo_d); /* u3_disk_epoc_init(): create new epoch. */ From 3b1ee9a0d2b8c5217c16b7bae563c924a7aae751 Mon Sep 17 00:00:00 2001 From: Matthew LeVan Date: Thu, 25 May 2023 15:08:04 -0400 Subject: [PATCH 027/271] cli: fix `chop` bug --- pkg/vere/disk.c | 1 + pkg/vere/main.c | 64 +++++++++++++++++++++++++++++++++++++++---------- 2 files changed, 52 insertions(+), 13 deletions(-) diff --git a/pkg/vere/disk.c b/pkg/vere/disk.c index 3985528493..ce6930ea89 100644 --- a/pkg/vere/disk.c +++ b/pkg/vere/disk.c @@ -1224,6 +1224,7 @@ c3_o u3_disk_migrate(u3_disk* log_u) * 0. detect whether we need to migrate or not * a. if it's a fresh boot via u3_Host.ops_u.nuu -> skip migration * b. if data.mdb is readable in log directory -> execute migration + * if not -> skip migration * 1. initialize epoch 0i0 (first call to u3_disk_epoc_init()) * a. creates epoch directory * b. creates epoch version file diff --git a/pkg/vere/main.c b/pkg/vere/main.c index c3da6be457..7f9d360e9a 100644 --- a/pkg/vere/main.c +++ b/pkg/vere/main.c @@ -2157,15 +2157,48 @@ _cw_chop(c3_i argc, c3_c* argv[]) fprintf(stderr, "chop: failed to get first/last events\r\n"); exit(1); } - if ( fir_d == las_d ) { - fprintf(stderr, "chop: latest epoch already empty\r\n"); - exit(1); - } - else if ( c3n == u3_disk_epoc_init(log_u) ) { + + // create new epoch if latest isn't empty + if ( (fir_d != las_d) && (c3n == u3_disk_epoc_init(log_u)) ) { fprintf(stderr, "chop: failed to create new epoch\r\n"); exit(1); } + // sort epoch directories in descending order + u3_dire* ned_u = u3_foil_folder(log_u->com_u->pax_c); + u3_dent* den_u = ned_u->dil_u; + c3_z len_z = 0; + while ( den_u ) { // count epochs + len_z++; + den_u = den_u->nex_u; + } + c3_d* sot_d = c3_malloc(len_z * sizeof(c3_d)); + len_z = 0; + den_u = ned_u->dil_u; + while ( den_u ) { + if ( 1 == sscanf(den_u->nam_c, "0i%" PRIu64, (sot_d + len_z)) ) { + len_z++; + } + den_u = den_u->nex_u; + } + + if ( len_z <= 2 ) { + fprintf(stderr, "chop: nothing to do, have a great day\r\n"); + exit(0); // enjoy + } + + // sort sot_d naively in descending order + c3_d tmp_d; + for ( c3_z i_z = 0; i_z < len_z; i_z++ ) { + for ( c3_z j_z = i_z + 1; j_z < len_z; j_z++ ) { + if ( sot_d[i_z] < sot_d[j_z] ) { + tmp_d = sot_d[i_z]; + sot_d[i_z] = sot_d[j_z]; + sot_d[j_z] = tmp_d; + } + } + } + // get latest epoch number prior to creating a new one c3_d pos_d; if ( c3n == u3_disk_epoc_last(log_u, &pos_d) ) { @@ -2174,20 +2207,25 @@ _cw_chop(c3_i argc, c3_c* argv[]) } // delete all but the last two epochs - u3_dent* den_u = log_u->com_u->dil_u; + // XX parameterize the number of epochs to chop + for ( c3_z i_z = 2; i_z < len_z; i_z++ ) { + fprintf(stderr, "chop: deleting epoch 0i%" PRIu64 "\r\n", sot_d[i_z]); + if ( c3y != u3_disk_epoc_kill(log_u, sot_d[i_z]) ) { + fprintf(stderr, "chop: failed to delete epoch 0i%" PRIu64 "\r\n", sot_d[i_z]); + exit(1); + } + } + c3_d epo_d = 0; while ( den_u ) { - c3_d res_d = sscanf(den_u->nam_c, "0i%" PRIu64, &epo_d); - if ( (1 == res_d) && (epo_d != pre_d) && (epo_d != pos_d) ) { - fprintf(stderr, "chop: deleting epoch 0i%" PRIu64 "\r\n", epo_d); - if ( c3y != u3_disk_epoc_kill(log_u, epo_d) ) { - fprintf(stderr, "chop: failed to delete epoch 0i%" PRIu64 "\r\n", epo_d); - } + c3_i res_i = sscanf(den_u->nam_c, "0i%" PRIu64, &epo_d); + if ( (1 == res_i) && (epo_d < pre_d) && (epo_d < pos_d) ) { } den_u = den_u->nex_u; } // cleanup + u3_dire_free(ned_u); u3_disk_exit(log_u); u3m_stop(); @@ -2269,7 +2307,7 @@ _cw_roll(c3_i argc, c3_c* argv[]) if ( fir_d == las_d ) { fprintf(stderr, "roll: latest epoch already empty\r\n"); - exit(1); + exit(0); } else if ( c3n == u3_disk_epoc_init(log_u) ) { fprintf(stderr, "roll: failed to create new epoch\r\n"); From b8b52c239e4f2c3de03a119999a6c7408039f8f3 Mon Sep 17 00:00:00 2001 From: Matthew LeVan Date: Fri, 26 May 2023 09:43:55 -0400 Subject: [PATCH 028/271] disk: move format version to `version.h` --- pkg/noun/version.h | 6 ++++++ pkg/vere/disk.c | 6 ------ 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/pkg/noun/version.h b/pkg/noun/version.h index f64d0a398e..125abd684c 100644 --- a/pkg/noun/version.h +++ b/pkg/noun/version.h @@ -18,4 +18,10 @@ typedef c3_w u3e_version; #define U3E_VER1 1 #define U3E_VERLAT U3E_VER1 +/* DISK FORMAT + */ + +#define U3D_VER1 1 +#define U3D_VERLAT U3L_VER1 + #endif /* ifndef U3_VERSION_H */ diff --git a/pkg/vere/disk.c b/pkg/vere/disk.c index ce6930ea89..2138ed4205 100644 --- a/pkg/vere/disk.c +++ b/pkg/vere/disk.c @@ -23,12 +23,6 @@ struct _cd_save { struct _u3_disk* log_u; }; -/* DISK FORMAT - */ - -#define U3D_VER1 1 -#define U3D_VERLAT U3L_VER1 - #undef VERBOSE_DISK #undef DISK_TRACE_JAM #undef DISK_TRACE_CUE From 6582f14ad1be153d2ed636ae8077efd90fc26a03 Mon Sep 17 00:00:00 2001 From: Matthew LeVan Date: Tue, 30 May 2023 11:47:23 -0400 Subject: [PATCH 029/271] disk: use `PRIc3_d` instead of `PRIu64` --- pkg/vere/disk.c | 36 ++++++++++++++++-------------------- pkg/vere/main.c | 10 +++++----- 2 files changed, 21 insertions(+), 25 deletions(-) diff --git a/pkg/vere/disk.c b/pkg/vere/disk.c index 2138ed4205..27c7b14f66 100644 --- a/pkg/vere/disk.c +++ b/pkg/vere/disk.c @@ -905,7 +905,7 @@ u3_disk_init(c3_c* pax_c, u3_disk_cb cb_u) // set path to latest epoch c3_c epo_c[8193]; - snprintf(epo_c, 8192, "%s/0i%" PRIu64, log_c, lat_d); + snprintf(epo_c, 8192, "%s/0i%" PRIc3_d, log_c, lat_d); // initialize latest epoch's db if ( 0 == (log_u->mdb_u = u3_lmdb_init(epo_c)) ) { @@ -913,7 +913,7 @@ u3_disk_init(c3_c* pax_c, u3_disk_cb cb_u) c3_free(log_u); return 0; } - fprintf(stderr, "disk: loaded epoch 0i%" PRIu64 "\r\n", lat_d); + fprintf(stderr, "disk: loaded epoch 0i%" PRIc3_d "\r\n", lat_d); // get first/last event numbers from lmdb c3_d fir_d, las_d; @@ -975,7 +975,7 @@ c3_o u3_disk_epoc_good(u3_disk* log_u, c3_d epo_d) { // file paths c3_c epo_c[8193], dat_c[8193], epv_c[8193], biv_c[8193]; - snprintf(epo_c, sizeof(epo_c), "%s/0i%" PRIu64, log_u->com_u->pax_c, epo_d); + snprintf(epo_c, sizeof(epo_c), "%s/0i%" PRIc3_d, log_u->com_u->pax_c, epo_d); snprintf(dat_c, sizeof(dat_c), "%s/data.mdb", epo_c); snprintf(epv_c, sizeof(epv_c), "%s/epoc.txt", epo_c); snprintf(biv_c, sizeof(biv_c), "%s/vere.txt", epo_c); @@ -1014,23 +1014,23 @@ c3_o u3_disk_epoc_good(u3_disk* log_u, c3_d epo_d) { // print error messages if ( c3n == dir_o ) { - fprintf(stderr, "disk: epoch 0i%" PRIu64 " is not writable\r\n", epo_d); + fprintf(stderr, "disk: epoch 0i%" PRIc3_d " is not writable\r\n", epo_d); ret_o = c3n; } if ( c3n == dat_o ) { - fprintf(stderr, "disk: epoch 0i%" PRIu64 "/data.mdb is not writable\r\n", epo_d); + fprintf(stderr, "disk: epoch 0i%" PRIc3_d "/data.mdb is not writable\r\n", epo_d); ret_o = c3n; } if ( c3n == mdb_o ) { - fprintf(stderr, "disk: epoch 0i%" PRIu64 "/data.mdb can't be opened\r\n", epo_d); + fprintf(stderr, "disk: epoch 0i%" PRIc3_d "/data.mdb can't be opened\r\n", epo_d); ret_o = c3n; } if ( c3n == epv_o ) { - fprintf(stderr, "disk: epoch 0i%" PRIu64 "/epoc.txt is not writable\r\n", epo_d); + fprintf(stderr, "disk: epoch 0i%" PRIc3_d "/epoc.txt is not writable\r\n", epo_d); ret_o = c3n; } if ( c3n == biv_o ) { - fprintf(stderr, "disk: epoch 0i%" PRIu64 "/vere.txt is not writable\r\n", epo_d); + fprintf(stderr, "disk: epoch 0i%" PRIc3_d "/vere.txt is not writable\r\n", epo_d); ret_o = c3n; } @@ -1063,10 +1063,10 @@ c3_o u3_disk_epoc_init(u3_disk* log_u) { // create new epoch directory if it doesn't exist c3_c epo_c[8193]; - snprintf(epo_c, sizeof(epo_c), "%s/0i%" PRIu64, log_u->com_u->pax_c, new_d); + snprintf(epo_c, sizeof(epo_c), "%s/0i%" PRIc3_d, log_u->com_u->pax_c, new_d); c3_d ret_d = c3_mkdir(epo_c, 0700); if ( ( ret_d < 0 ) && ( errno != EEXIST ) ) { - fprintf(stderr, "disk: failed to create epoch directory %" PRIu64 "\r\n", new_d); + fprintf(stderr, "disk: failed to create epoch directory %" PRIc3_d "\r\n", new_d); return c3n; } @@ -1107,10 +1107,6 @@ c3_o u3_disk_epoc_init(u3_disk* log_u) { if ( c3y == u3_Host.ops_u.nuu || new_d > 0 ) { c3_c dat_c[8193]; snprintf(dat_c, sizeof(dat_c), "%s/data.mdb", epo_c); - // if ( c3n == c3_unlink(dat_c) ) { - // fprintf(stderr, "disk: failed to rm 0i%" PRIu64 "/data.mdb\r\n", new_d); - // goto fail; - // }; if ( 0 == (log_u->mdb_u = u3_lmdb_init(epo_c)) ) { fprintf(stderr, "disk: failed to initialize database\r\n"); c3_free(log_u); @@ -1144,7 +1140,7 @@ c3_o u3_disk_epoc_init(u3_disk* log_u) { c3_o u3_disk_epoc_kill(u3_disk* log_u, c3_d epo_d) { // get epoch directory c3_c epo_c[8193]; - snprintf(epo_c, sizeof(epo_c), "%s/0i%" PRIu64, log_u->com_u->pax_c, epo_d); + snprintf(epo_c, sizeof(epo_c), "%s/0i%" PRIc3_d, log_u->com_u->pax_c, epo_d); // delete files in epoch directory u3_dire* dir_u = u3_foil_folder(epo_c); @@ -1177,7 +1173,7 @@ c3_o u3_disk_epoc_last(u3_disk* log_u, c3_d* lat_d) { u3_dent* den_u = u3_foil_folder(log_u->com_u->pax_c)->dil_u; while ( den_u ) { c3_d epo_d = 0; - if ( 1 == sscanf(den_u->nam_c, "0i%" PRIu64, &epo_d) ) { + if ( 1 == sscanf(den_u->nam_c, "0i%" PRIc3_d, &epo_d) ) { ret_d = c3y; // NB: returns yes if the directory merely exists } *lat_d = c3_max(epo_d, *lat_d); // update the latest epoch number @@ -1192,18 +1188,18 @@ c3_o u3_disk_epoc_last(u3_disk* log_u, c3_d* lat_d) { c3_o u3_disk_epoc_vere(u3_disk* log_u, c3_d epo_d, c3_c* ver_w) { c3_c ver_c[8193]; - snprintf(ver_c, sizeof(ver_c), "%s/0i%" PRIu64 "/vere.txt", + snprintf(ver_c, sizeof(ver_c), "%s/0i%" PRIc3_d "/vere.txt", log_u->com_u->pax_c, epo_d); FILE* fil_u = fopen(ver_c, "r"); if ( NULL == fil_u ) { - fprintf(stderr, "disk: failed to open vere.txt in epoch 0i%" PRIu64 + fprintf(stderr, "disk: failed to open vere.txt in epoch 0i%" PRIc3_d "\r\n", epo_d); return c3n; } if ( 1 != fscanf(fil_u, "%s", ver_w) ) { - fprintf(stderr, "disk: failed to read vere.txt in epoch 0i%" PRIu64 + fprintf(stderr, "disk: failed to read vere.txt in epoch 0i%" PRIc3_d "\r\n", epo_d); return c3n; } @@ -1283,7 +1279,7 @@ c3_o u3_disk_migrate(u3_disk* log_u) if ( u3_Host.eve_d != las_d ) { fprintf(stderr, "disk: migrate: error: snapshot is out of date, please " "start/shutdown your pier gracefully first\r\n"); - fprintf(stderr, "disk: migrate: eve_d (%" PRIu64 ") != las_d (%" PRIu64 ")\r\n", + fprintf(stderr, "disk: migrate: eve_d (%" PRIc3_d ") != las_d (%" PRIc3_d ")\r\n", u3_Host.eve_d, las_d); return c3n; } diff --git a/pkg/vere/main.c b/pkg/vere/main.c index 7f9d360e9a..71bd981cbb 100644 --- a/pkg/vere/main.c +++ b/pkg/vere/main.c @@ -2140,7 +2140,7 @@ _cw_chop(c3_i argc, c3_c* argv[]) if ( log_u->dun_d != u3A->eve_d ) { fprintf(stderr, "chop: error: snapshot is out of date, please " "start/shutdown your pier gracefully first\r\n"); - fprintf(stderr, "chop: eve_d: %" PRIu64 ", dun_d: %" PRIu64 "\r\n", u3A->eve_d, log_u->dun_d); + fprintf(stderr, "chop: eve_d: %" PRIc3_d ", dun_d: %" PRIc3_d "\r\n", u3A->eve_d, log_u->dun_d); exit(1); } @@ -2176,7 +2176,7 @@ _cw_chop(c3_i argc, c3_c* argv[]) len_z = 0; den_u = ned_u->dil_u; while ( den_u ) { - if ( 1 == sscanf(den_u->nam_c, "0i%" PRIu64, (sot_d + len_z)) ) { + if ( 1 == sscanf(den_u->nam_c, "0i%" PRIc3_d, (sot_d + len_z)) ) { len_z++; } den_u = den_u->nex_u; @@ -2209,7 +2209,7 @@ _cw_chop(c3_i argc, c3_c* argv[]) // delete all but the last two epochs // XX parameterize the number of epochs to chop for ( c3_z i_z = 2; i_z < len_z; i_z++ ) { - fprintf(stderr, "chop: deleting epoch 0i%" PRIu64 "\r\n", sot_d[i_z]); + fprintf(stderr, "chop: deleting epoch 0i%" PRIc3_d "\r\n", sot_d[i_z]); if ( c3y != u3_disk_epoc_kill(log_u, sot_d[i_z]) ) { fprintf(stderr, "chop: failed to delete epoch 0i%" PRIu64 "\r\n", sot_d[i_z]); exit(1); @@ -2218,7 +2218,7 @@ _cw_chop(c3_i argc, c3_c* argv[]) c3_d epo_d = 0; while ( den_u ) { - c3_i res_i = sscanf(den_u->nam_c, "0i%" PRIu64, &epo_d); + c3_i res_i = sscanf(den_u->nam_c, "0i%" PRIc3_d, &epo_d); if ( (1 == res_i) && (epo_d < pre_d) && (epo_d < pos_d) ) { } den_u = den_u->nex_u; @@ -2293,7 +2293,7 @@ _cw_roll(c3_i argc, c3_c* argv[]) if ( log_u->dun_d != u3A->eve_d ) { fprintf(stderr, "roll: error: snapshot is out of date, please " "start/shutdown your pier gracefully first\r\n"); - fprintf(stderr, "roll: eve_d: %" PRIu64 ", dun_d: %" PRIu64 "\r\n", \ + fprintf(stderr, "roll: eve_d: %" PRIc3_d ", dun_d: %" PRIc3_d "\r\n", \ u3A->eve_d, log_u->dun_d); exit(1); } From b35b413f9aaabd71924524657d91cc03dcf70144 Mon Sep 17 00:00:00 2001 From: Matthew LeVan Date: Wed, 31 May 2023 15:34:17 -0400 Subject: [PATCH 030/271] disk: various epoch improvements --- pkg/noun/manage.c | 1 + pkg/vere/disk.c | 43 ++++++++++++++++++++++++++----------------- pkg/vere/main.c | 11 ++--------- 3 files changed, 29 insertions(+), 26 deletions(-) diff --git a/pkg/noun/manage.c b/pkg/noun/manage.c index 166dcdf8df..2d467c11ce 100644 --- a/pkg/noun/manage.c +++ b/pkg/noun/manage.c @@ -2054,6 +2054,7 @@ extern void u3je_secp_stop(void); void u3m_stop() { + // XX make sure to cleanup snapshot file descriptors if necessary u3je_secp_stop(); } diff --git a/pkg/vere/disk.c b/pkg/vere/disk.c index 27c7b14f66..6f7ce2176f 100644 --- a/pkg/vere/disk.c +++ b/pkg/vere/disk.c @@ -896,8 +896,8 @@ u3_disk_init(c3_c* pax_c, u3_disk_cb cb_u) } // get binary version from latest epoch - c3_c ver_w[8193]; - if ( c3n == u3_disk_epoc_vere(log_u, lat_d, ver_w) ) { + c3_c ver_c[8193]; + if ( c3n == u3_disk_epoc_vere(log_u, lat_d, ver_c) ) { fprintf(stderr, "disk: failed to load epoch version\r\n"); c3_free(log_u); return 0; @@ -936,7 +936,7 @@ u3_disk_init(c3_c* pax_c, u3_disk_cb cb_u) // if binary version of latest epoch is not the same as the // running binary, then we need to create a new epoch - if ( 0 != strcmp(ver_w, URBIT_VERSION) ) { + if ( 0 != strcmp(ver_c, URBIT_VERSION) ) { if ( c3n == u3_disk_epoc_init(log_u) ) { fprintf(stderr, "disk: failed to initialize epoch\r\n"); c3_free(log_u); @@ -1161,6 +1161,9 @@ c3_o u3_disk_epoc_kill(u3_disk* log_u, c3_d epo_d) { return c3n; } + // cleanup + u3_dire_free(dir_u); + // success return c3y; } @@ -1168,25 +1171,31 @@ c3_o u3_disk_epoc_kill(u3_disk* log_u, c3_d epo_d) { /* u3_disk_epoc_last: get latest epoch number. */ c3_o u3_disk_epoc_last(u3_disk* log_u, c3_d* lat_d) { - c3_o ret_d = c3n; // return no if no epoch directories exist + c3_o ret_o = c3n; // return no if no epoch directories exist *lat_d = 0; // initialize lat_d to 0 - u3_dent* den_u = u3_foil_folder(log_u->com_u->pax_c)->dil_u; + u3_dire* die_u = u3_foil_folder(log_u->com_u->pax_c); + u3_dent* den_u = die_u->dil_u; while ( den_u ) { c3_d epo_d = 0; if ( 1 == sscanf(den_u->nam_c, "0i%" PRIc3_d, &epo_d) ) { - ret_d = c3y; // NB: returns yes if the directory merely exists + ret_o = c3y; // NB: returns yes if the directory merely exists + *lat_d = c3_max(epo_d, *lat_d); // update the latest epoch number } - *lat_d = c3_max(epo_d, *lat_d); // update the latest epoch number den_u = den_u->nex_u; } + + u3_dire_free(die_u); - return ret_d; + return ret_o; } /* u3_disk_epoc_vere: get binary version from epoch. */ c3_o u3_disk_epoc_vere(u3_disk* log_u, c3_d epo_d, c3_c* ver_w) { + // XX check implementation (similar code is in king.c, _king_get_pace) + // should probably write a generic function to read a string from + // a file into malloc'd memory, with an optional whitespace trimming c3_c ver_c[8193]; snprintf(ver_c, sizeof(ver_c), "%s/0i%" PRIc3_d "/vere.txt", log_u->com_u->pax_c, epo_d); @@ -1210,12 +1219,12 @@ u3_disk_epoc_vere(u3_disk* log_u, c3_d epo_d, c3_c* ver_w) { */ c3_o u3_disk_migrate(u3_disk* log_u) { - /* migration steps: + /* migration steps (* indicates breakpoint set): * 0. detect whether we need to migrate or not - * a. if it's a fresh boot via u3_Host.ops_u.nuu -> skip migration - * b. if data.mdb is readable in log directory -> execute migration - * if not -> skip migration - * 1. initialize epoch 0i0 (first call to u3_disk_epoc_init()) + * a. if it's a fresh boot via u3_Host.ops_u.nuu -> skip migration (returns yes) + * b. if data.mdb is readable in log directory -> execute migration (returns yes or no) + * if not -> skip migration (returns yes) + * 1. initialize epoch 0i0 (first call to u3_disk_epoc_init()) * * a. creates epoch directory * b. creates epoch version file * c. creates binary version file @@ -1224,10 +1233,10 @@ c3_o u3_disk_migrate(u3_disk* log_u) * f. writes metadata to new database * g. loads new epoch directory and sets it in log_u * 2. create hard links to data.mdb and lock.mdb in 0i0/ - * 3. rollover to new epoch (second call to u3_disk_epoc_init()) + * 3. rollover to new epoch (second call to u3_disk_epoc_init()) * * a. same as 1a-g but also copies current snapshot between c/d steps - * 4. delete backup snapshot (c3_unlink() and c3_rmdir() calls) - * 5. delete old data.mdb and lock.mdb files (c3_unlink() calls) + * 4. delete backup snapshot (c3_unlink() and c3_rmdir() calls) * + * 5. delete old data.mdb and lock.mdb files (c3_unlink() calls) * */ // check if data.mdb is readable in log directory @@ -1252,7 +1261,7 @@ c3_o u3_disk_migrate(u3_disk* log_u) // initialize first epoch "0i0" if ( c3n == u3_disk_epoc_init(log_u) ) { - fprintf(stderr, "disk: migrate: failed to initialize first epoch\r\n"); + fprintf(stderr, "disk: failed to initialize first epoch\r\n"); return c3n; } } else if ( c3y == dat_o ) { diff --git a/pkg/vere/main.c b/pkg/vere/main.c index 71bd981cbb..79a1a0fe92 100644 --- a/pkg/vere/main.c +++ b/pkg/vere/main.c @@ -2216,14 +2216,6 @@ _cw_chop(c3_i argc, c3_c* argv[]) } } - c3_d epo_d = 0; - while ( den_u ) { - c3_i res_i = sscanf(den_u->nam_c, "0i%" PRIc3_d, &epo_d); - if ( (1 == res_i) && (epo_d < pre_d) && (epo_d < pos_d) ) { - } - den_u = den_u->nex_u; - } - // cleanup u3_dire_free(ned_u); u3_disk_exit(log_u); @@ -2769,7 +2761,8 @@ main(c3_i argc, // validate whether we can execute disk migration if ( u3_Host.ops_u.nuu == c3n ) { u3_Host.eve_d = u3m_boot(u3_Host.dir_c, (size_t)1 << u3_Host.ops_u.lom_y); - // XX u3e_close() + u3m_stop(); + // XX unmap loom, else parts of the snapshot could be left in memory } // starting u3m configures OpenSSL memory functions, so we must do it From ba1a3af7f1f7ee122e7cdb56382975730b701b32 Mon Sep 17 00:00:00 2001 From: Matthew LeVan Date: Wed, 31 May 2023 18:01:43 -0400 Subject: [PATCH 031/271] disk: function style fixes --- pkg/vere/disk.c | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/pkg/vere/disk.c b/pkg/vere/disk.c index 6f7ce2176f..9a5f8dfacc 100644 --- a/pkg/vere/disk.c +++ b/pkg/vere/disk.c @@ -958,7 +958,8 @@ u3_disk_init(c3_c* pax_c, u3_disk_cb cb_u) /* u3_disk_epoc_good: check for valid epoch. */ -c3_o u3_disk_epoc_good(u3_disk* log_u, c3_d epo_d) { +c3_o +u3_disk_epoc_good(u3_disk* log_u, c3_d epo_d) { /* a "valid" epoch is currently defined as a writable folder in * the /.urb/log directory named with a @ui and contains: * - a writable data.mdb file which can be opened by lmdb (add db checking?) @@ -1039,7 +1040,8 @@ c3_o u3_disk_epoc_good(u3_disk* log_u, c3_d epo_d) { /* u3_disk_epoc_init: create new epoch. */ -c3_o u3_disk_epoc_init(u3_disk* log_u) { +c3_o +u3_disk_epoc_init(u3_disk* log_u) { // set new epoch number c3_d lat_d, new_d; c3_o eps_o = u3_disk_epoc_last(log_u, &lat_d); @@ -1137,7 +1139,8 @@ c3_o u3_disk_epoc_init(u3_disk* log_u) { /* u3_disk_epoc_kill: delete an epoch. */ -c3_o u3_disk_epoc_kill(u3_disk* log_u, c3_d epo_d) { +c3_o +u3_disk_epoc_kill(u3_disk* log_u, c3_d epo_d) { // get epoch directory c3_c epo_c[8193]; snprintf(epo_c, sizeof(epo_c), "%s/0i%" PRIc3_d, log_u->com_u->pax_c, epo_d); @@ -1170,7 +1173,8 @@ c3_o u3_disk_epoc_kill(u3_disk* log_u, c3_d epo_d) { /* u3_disk_epoc_last: get latest epoch number. */ -c3_o u3_disk_epoc_last(u3_disk* log_u, c3_d* lat_d) { +c3_o +u3_disk_epoc_last(u3_disk* log_u, c3_d* lat_d) { c3_o ret_o = c3n; // return no if no epoch directories exist *lat_d = 0; // initialize lat_d to 0 u3_dire* die_u = u3_foil_folder(log_u->com_u->pax_c); @@ -1358,4 +1362,4 @@ c3_o u3_disk_migrate(u3_disk* log_u) } return c3y; -} \ No newline at end of file +} From 9b45ad20ec0d0f94437b6afe2ec8eec387459687 Mon Sep 17 00:00:00 2001 From: Matthew LeVan Date: Thu, 1 Jun 2023 09:14:16 -0400 Subject: [PATCH 032/271] u3: remove snapshot backup in `u3e_save` and unused `u3m_backup` --- pkg/noun/events.c | 4 ---- pkg/noun/manage.c | 10 ---------- 2 files changed, 14 deletions(-) diff --git a/pkg/noun/events.c b/pkg/noun/events.c index 5e16182307..287ac92d82 100644 --- a/pkg/noun/events.c +++ b/pkg/noun/events.c @@ -1379,10 +1379,6 @@ u3e_save(u3_post low_p, u3_post hig_p) pgs_w, ptr_v, strerror(errno)); } } - - c3_c bhk_c[8193]; - snprintf(bhk_c, sizeof(bhk_c), "%s/.urb/bhk", u3P.dir_c); - u3e_backup(bhk_c, c3n); } /* u3e_live(): start the checkpointing system. diff --git a/pkg/noun/manage.c b/pkg/noun/manage.c index 2d467c11ce..ee93ed5568 100644 --- a/pkg/noun/manage.c +++ b/pkg/noun/manage.c @@ -1748,16 +1748,6 @@ _cm_limits(void) # endif } -/* u3m_backup(): copy snapshot to .urb/bhk (if it doesn't exist yet). -*/ -c3_o -u3m_backup(c3_o ovw_o) -{ - c3_c bhk_c[2048]; - snprintf(bhk_c, 2048, "%s/.urb/bhk", u3P.dir_c); - return u3e_backup(bhk_c, ovw_o); -} - /* u3m_fault(): handle a memory event with libsigsegv protocol. */ c3_i From fb762b2fb24846a8a4e50d852c4ca18149df3c9c Mon Sep 17 00:00:00 2001 From: Matthew LeVan Date: Thu, 1 Jun 2023 09:40:45 -0400 Subject: [PATCH 033/271] disk: add event check to `u3_disk_epoc_good` --- pkg/vere/disk.c | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/pkg/vere/disk.c b/pkg/vere/disk.c index 9a5f8dfacc..70661b6634 100644 --- a/pkg/vere/disk.c +++ b/pkg/vere/disk.c @@ -983,6 +983,7 @@ u3_disk_epoc_good(u3_disk* log_u, c3_d epo_d) { c3_o dir_o = c3n; // directory is writable c3_o dat_o = c3n; // data.mdb is writable + c3_o len_o = c3n; // data.mdb has events c3_o mdb_o = c3n; // data.mdb can be opened c3_o epv_o = c3n; // epoc.txt is writable c3_o biv_o = c3n; // vere.txt is writable @@ -1000,6 +1001,17 @@ u3_disk_epoc_good(u3_disk* log_u, c3_d epo_d) { if ( 0 != (env_u = u3_lmdb_init(epo_c)) ) { mdb_o = c3y; } + + // check if there are any events in the database + c3_d low_d, hig_d; + if ( 0 != u3_lmdb_gulf(env_u, &low_d, &hig_d) ) { + fprintf(stderr, "disk: failed to get first/last event numbers\r\n"); + ret_o = c3n; + } else { + if ( low_d == hig_d == 0 ) { + len_o = c3y; // yes, data.mdb has events + } + } u3_lmdb_exit(env_u); } @@ -1026,6 +1038,10 @@ u3_disk_epoc_good(u3_disk* log_u, c3_d epo_d) { fprintf(stderr, "disk: epoch 0i%" PRIc3_d "/data.mdb can't be opened\r\n", epo_d); ret_o = c3n; } + if ( c3n == len_o ) { + fprintf(stderr, "disk: epoch 0i%" PRIc3_d "/data.mdb has no events\r\n", epo_d); + ret_o = c3n; + } if ( c3n == epv_o ) { fprintf(stderr, "disk: epoch 0i%" PRIc3_d "/epoc.txt is not writable\r\n", epo_d); ret_o = c3n; From 83b57376bcfc4a54d9959ebc5e027b68d268daf1 Mon Sep 17 00:00:00 2001 From: barter-simsum Date: Thu, 1 Jun 2023 10:18:14 -0400 Subject: [PATCH 034/271] cleanup whitespace --- pkg/noun/events.c | 2 +- pkg/vere/disk.c | 22 +++++++++++----------- pkg/vere/main.c | 4 ++-- pkg/vere/vere.h | 10 +++++----- 4 files changed, 19 insertions(+), 19 deletions(-) diff --git a/pkg/noun/events.c b/pkg/noun/events.c index 287ac92d82..89013cecd9 100644 --- a/pkg/noun/events.c +++ b/pkg/noun/events.c @@ -1220,7 +1220,7 @@ u3e_backup(c3_c* pax_c, c3_o ovw_o) { u3e_image nop_u = { .nam_c = "north", .pgs_w = 0 }; u3e_image sop_u = { .nam_c = "south", .pgs_w = 0 }; - + c3_i mod_i = O_RDWR | O_CREAT; if ( !pax_c ) { diff --git a/pkg/vere/disk.c b/pkg/vere/disk.c index 70661b6634..6e18eebef9 100644 --- a/pkg/vere/disk.c +++ b/pkg/vere/disk.c @@ -929,8 +929,8 @@ u3_disk_init(c3_c* pax_c, u3_disk_cb cb_u) } else { // not first epoch log_u->dun_d = lat_d; // set dun_d to last event in prev epoch } - } else { // not fresh epoch - log_u->dun_d = las_d; // set dun_d to last event in lmdb + } else { // not fresh epoch + log_u->dun_d = las_d; // set dun_d to last event in lmdb } log_u->sen_d = log_u->dun_d; @@ -960,7 +960,7 @@ u3_disk_init(c3_c* pax_c, u3_disk_cb cb_u) */ c3_o u3_disk_epoc_good(u3_disk* log_u, c3_d epo_d) { - /* a "valid" epoch is currently defined as a writable folder in + /* a "valid" epoch is currently defined as a writable folder in * the /.urb/log directory named with a @ui and contains: * - a writable data.mdb file which can be opened by lmdb (add db checking?) * - a writable epoc.txt file @@ -968,7 +968,7 @@ u3_disk_epoc_good(u3_disk* log_u, c3_d epo_d) { * * XX: should we check if the correct event sequence exists in the data.mdb file? * - * note that this does not check if the epoch even contains snapshot files, + * note that this does not check if the epoch even contains snapshot files, * let alone whether they're valid or not */ @@ -1053,7 +1053,7 @@ u3_disk_epoc_good(u3_disk* log_u, c3_d epo_d) { return ret_o; } - + /* u3_disk_epoc_init: create new epoch. */ c3_o @@ -1203,7 +1203,7 @@ u3_disk_epoc_last(u3_disk* log_u, c3_d* lat_d) { } den_u = den_u->nex_u; } - + u3_dire_free(die_u); return ret_o; @@ -1214,21 +1214,21 @@ u3_disk_epoc_last(u3_disk* log_u, c3_d* lat_d) { c3_o u3_disk_epoc_vere(u3_disk* log_u, c3_d epo_d, c3_c* ver_w) { // XX check implementation (similar code is in king.c, _king_get_pace) - // should probably write a generic function to read a string from + // should probably write a generic function to read a string from // a file into malloc'd memory, with an optional whitespace trimming c3_c ver_c[8193]; - snprintf(ver_c, sizeof(ver_c), "%s/0i%" PRIc3_d "/vere.txt", + snprintf(ver_c, sizeof(ver_c), "%s/0i%" PRIc3_d "/vere.txt", log_u->com_u->pax_c, epo_d); FILE* fil_u = fopen(ver_c, "r"); if ( NULL == fil_u ) { - fprintf(stderr, "disk: failed to open vere.txt in epoch 0i%" PRIc3_d + fprintf(stderr, "disk: failed to open vere.txt in epoch 0i%" PRIc3_d "\r\n", epo_d); return c3n; } if ( 1 != fscanf(fil_u, "%s", ver_w) ) { - fprintf(stderr, "disk: failed to read vere.txt in epoch 0i%" PRIc3_d + fprintf(stderr, "disk: failed to read vere.txt in epoch 0i%" PRIc3_d "\r\n", epo_d); return c3n; } @@ -1286,7 +1286,7 @@ c3_o u3_disk_migrate(u3_disk* log_u) } } else if ( c3y == dat_o ) { // migrate existing pier which has either: - // - not started the migration, or + // - not started the migration, or // - crashed before completing the migration // initialize pre-migrated lmdb diff --git a/pkg/vere/main.c b/pkg/vere/main.c index 79a1a0fe92..cacb37d285 100644 --- a/pkg/vere/main.c +++ b/pkg/vere/main.c @@ -317,7 +317,7 @@ _main_getopt(c3_i argc, c3_c** argv) return c3n; } else { u3_Host.ops_u.sap_w = arg_w * 60; - if ( 0 == u3_Host.ops_u.sap_w) + if ( 0 == u3_Host.ops_u.sap_w) return c3n; } break; @@ -2757,7 +2757,7 @@ main(c3_i argc, } } - // we need the current snapshot's latest event number to + // we need the current snapshot's latest event number to // validate whether we can execute disk migration if ( u3_Host.ops_u.nuu == c3n ) { u3_Host.eve_d = u3m_boot(u3_Host.dir_c, (size_t)1 << u3_Host.ops_u.lom_y); diff --git a/pkg/vere/vere.h b/pkg/vere/vere.h index fbb28c1ef7..f3e47a98b6 100644 --- a/pkg/vere/vere.h +++ b/pkg/vere/vere.h @@ -988,27 +988,27 @@ */ c3_o u3_disk_epoc_good(u3_disk* log_u, c3_d epo_d); - + /* u3_disk_epoc_init(): create new epoch. */ c3_o u3_disk_epoc_init(u3_disk* log_u); - + /* u3_disk_epoc_kill(): delete an epoch. */ c3_o u3_disk_epoc_kill(u3_disk* log_u, c3_d epo_d); - + /* u3_disk_epoc_last(): get latest epoch number. */ c3_o u3_disk_epoc_last(u3_disk* log_u, c3_d* lat_d); - + /* u3_disk_epoc_vere(): get binary version from epoch. */ c3_o u3_disk_epoc_vere(u3_disk* log_u, c3_d epo_d, c3_c* ver_w); - + /* u3_disk_migrate(): migrates disk format. */ c3_o From 6dc1bd6e0a8537e2ba6c3eedf97ee68b85139fc4 Mon Sep 17 00:00:00 2001 From: barter-simsum Date: Thu, 1 Jun 2023 11:21:45 -0400 Subject: [PATCH 035/271] data.mdb -> lock.mdb --- pkg/vere/disk.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/vere/disk.c b/pkg/vere/disk.c index 6e18eebef9..56383a76b2 100644 --- a/pkg/vere/disk.c +++ b/pkg/vere/disk.c @@ -1270,7 +1270,7 @@ c3_o u3_disk_migrate(u3_disk* log_u) // check if lock.mdb is readable in log directory c3_o lok_o = c3n; c3_c lok_c[8193]; - snprintf(lok_c, sizeof(dat_c), "%s/data.mdb", log_u->com_u->pax_c); + snprintf(lok_c, sizeof(dat_c), "%s/lock.mdb", log_u->com_u->pax_c); if ( 0 == access(dat_c, R_OK) ) { lok_o = c3y; } From a0d239bb0d85f8084d96bb10092c6910b8ae873d Mon Sep 17 00:00:00 2001 From: Matthew LeVan Date: Thu, 1 Jun 2023 14:09:17 -0400 Subject: [PATCH 036/271] disk: add event check in `u3_disk_epoc_good` --- pkg/vere/disk.c | 178 +++++++++++++++++++++++++----------------------- 1 file changed, 93 insertions(+), 85 deletions(-) diff --git a/pkg/vere/disk.c b/pkg/vere/disk.c index 56383a76b2..f0807bbedf 100644 --- a/pkg/vere/disk.c +++ b/pkg/vere/disk.c @@ -959,15 +959,14 @@ u3_disk_init(c3_c* pax_c, u3_disk_cb cb_u) /* u3_disk_epoc_good: check for valid epoch. */ c3_o -u3_disk_epoc_good(u3_disk* log_u, c3_d epo_d) { +u3_disk_epoc_good(u3_disk* log_u, c3_d epo_d) +{ /* a "valid" epoch is currently defined as a writable folder in * the /.urb/log directory named with a @ui and contains: - * - a writable data.mdb file which can be opened by lmdb (add db checking?) + * - a writable data.mdb file which can be opened by lmdb and contains 1+ events * - a writable epoc.txt file * - a writable vere.txt file * - * XX: should we check if the correct event sequence exists in the data.mdb file? - * * note that this does not check if the epoch even contains snapshot files, * let alone whether they're valid or not */ @@ -1057,7 +1056,8 @@ u3_disk_epoc_good(u3_disk* log_u, c3_d epo_d) { /* u3_disk_epoc_init: create new epoch. */ c3_o -u3_disk_epoc_init(u3_disk* log_u) { +u3_disk_epoc_init(u3_disk* log_u) +{ // set new epoch number c3_d lat_d, new_d; c3_o eps_o = u3_disk_epoc_last(log_u, &lat_d); @@ -1156,7 +1156,8 @@ u3_disk_epoc_init(u3_disk* log_u) { /* u3_disk_epoc_kill: delete an epoch. */ c3_o -u3_disk_epoc_kill(u3_disk* log_u, c3_d epo_d) { +u3_disk_epoc_kill(u3_disk* log_u, c3_d epo_d) +{ // get epoch directory c3_c epo_c[8193]; snprintf(epo_c, sizeof(epo_c), "%s/0i%" PRIc3_d, log_u->com_u->pax_c, epo_d); @@ -1190,7 +1191,8 @@ u3_disk_epoc_kill(u3_disk* log_u, c3_d epo_d) { /* u3_disk_epoc_last: get latest epoch number. */ c3_o -u3_disk_epoc_last(u3_disk* log_u, c3_d* lat_d) { +u3_disk_epoc_last(u3_disk* log_u, c3_d* lat_d) +{ c3_o ret_o = c3n; // return no if no epoch directories exist *lat_d = 0; // initialize lat_d to 0 u3_dire* die_u = u3_foil_folder(log_u->com_u->pax_c); @@ -1212,7 +1214,8 @@ u3_disk_epoc_last(u3_disk* log_u, c3_d* lat_d) { /* u3_disk_epoc_vere: get binary version from epoch. */ c3_o -u3_disk_epoc_vere(u3_disk* log_u, c3_d epo_d, c3_c* ver_w) { +u3_disk_epoc_vere(u3_disk* log_u, c3_d epo_d, c3_c* ver_w) +{ // XX check implementation (similar code is in king.c, _king_get_pace) // should probably write a generic function to read a string from // a file into malloc'd memory, with an optional whitespace trimming @@ -1275,8 +1278,8 @@ c3_o u3_disk_migrate(u3_disk* log_u) lok_o = c3y; } + // if fresh boot, initialize disk v1 if ( c3y == u3_Host.ops_u.nuu ) { - // initialize disk v1 on fresh boots fprintf(stderr, "disk: initializing disk with v%d format\r\n", U3D_VER1); // initialize first epoch "0i0" @@ -1284,98 +1287,103 @@ c3_o u3_disk_migrate(u3_disk* log_u) fprintf(stderr, "disk: failed to initialize first epoch\r\n"); return c3n; } - } else if ( c3y == dat_o ) { - // migrate existing pier which has either: - // - not started the migration, or - // - crashed before completing the migration - - // initialize pre-migrated lmdb - { - if ( 0 == (log_u->mdb_u = u3_lmdb_init(log_u->com_u->pax_c)) ) { - fprintf(stderr, "disk: failed to initialize database\r\n"); - return c3n; - } - } - // get first/last event numbers from pre-migrated lmdb - c3_d fir_d, las_d; - if ( c3n == u3_lmdb_gulf(log_u->mdb_u, &fir_d, &las_d) ) { - fprintf(stderr, "disk: failed to get first/last event numbers\r\n"); - return c3n; - } + return c3y; + } + + // if data.mdb is not readable, skip migration + if ( c3n == dat_o ) return c3y; - // ensure there's a current snapshot - if ( u3_Host.eve_d != las_d ) { - fprintf(stderr, "disk: migrate: error: snapshot is out of date, please " - "start/shutdown your pier gracefully first\r\n"); - fprintf(stderr, "disk: migrate: eve_d (%" PRIc3_d ") != las_d (%" PRIc3_d ")\r\n", - u3_Host.eve_d, las_d); - return c3n; - } + // migrate existing pier which has either: + // - not started the migration, or + // - crashed before completing the migration - // initialize first epoch "0i0" - if ( c3n == u3_disk_epoc_init(log_u) ) { - fprintf(stderr, "disk: migrate: failed to initialize first epoch\r\n"); + // initialize pre-migrated lmdb + { + if ( 0 == (log_u->mdb_u = u3_lmdb_init(log_u->com_u->pax_c)) ) { + fprintf(stderr, "disk: failed to initialize database\r\n"); return c3n; } + } - // create hard links to data.mdb and lock.mdb in 0i0/ - c3_c dut_c[8193], luk_c[8193]; // old paths - snprintf(dut_c, sizeof(dut_c), "%s/data.mdb", log_u->com_u->pax_c); - snprintf(luk_c, sizeof(luk_c), "%s/lock.mdb", log_u->com_u->pax_c); + // get first/last event numbers from pre-migrated lmdb + c3_d fir_d, las_d; + if ( c3n == u3_lmdb_gulf(log_u->mdb_u, &fir_d, &las_d) ) { + fprintf(stderr, "disk: failed to get first/last event numbers\r\n"); + return c3n; + } - c3_c epo_c[8193], dat_c[8193], lok_c[8193]; // new paths - snprintf(epo_c, sizeof(epo_c), "%s/0i0", log_u->com_u->pax_c); - snprintf(dat_c, sizeof(dat_c), "%s/data.mdb", epo_c); - snprintf(lok_c, sizeof(lok_c), "%s/lock.mdb", epo_c); + // ensure there's a current snapshot + if ( u3_Host.eve_d != las_d ) { + fprintf(stderr, "disk: migrate: error: snapshot is out of date, please " + "start/shutdown your pier gracefully first\r\n"); + fprintf(stderr, "disk: migrate: eve_d (%" PRIc3_d ") != las_d (%" PRIc3_d ")\r\n", + u3_Host.eve_d, las_d); + return c3n; + } - if ( 0 < c3_link(dut_c, dat_c) ) { - fprintf(stderr, "disk: migrate: failed to create data.mdb hard link\r\n"); - fprintf(stderr, "errno %d: %s\r\n", errno, strerror(errno)); - return c3n; - } - if ( c3y == lok_o ) { // only link lock.mdb if it exists - if ( 0 < c3_link(luk_c, lok_c) ) { - fprintf(stderr, "disk: migrate: failed to create lock.mdb hard link\r\n"); - c3_rename(dat_c, dut_c); - return c3n; - } - } + // initialize first epoch "0i0" + if ( c3n == u3_disk_epoc_init(log_u) ) { + fprintf(stderr, "disk: migrate: failed to initialize first epoch\r\n"); + return c3n; + } - // rollover to new epoch - // log_u->mdb_u = old_u; - if ( c3n == u3_disk_epoc_init(log_u) ) { - fprintf(stderr, "roll: error: failed to initialize new epoch\r\n"); + // create hard links to data.mdb and lock.mdb in 0i0/ + c3_c dut_c[8193], luk_c[8193]; // old paths + snprintf(dut_c, sizeof(dut_c), "%s/data.mdb", log_u->com_u->pax_c); + snprintf(luk_c, sizeof(luk_c), "%s/lock.mdb", log_u->com_u->pax_c); + + c3_c epo_c[8193]; + snprintf(epo_c, sizeof(epo_c), "%s/0i0", log_u->com_u->pax_c); + snprintf(dat_c, sizeof(dat_c), "%s/data.mdb", epo_c); + snprintf(lok_c, sizeof(lok_c), "%s/lock.mdb", epo_c); + + if ( 0 < c3_link(dut_c, dat_c) ) { + fprintf(stderr, "disk: migrate: failed to create data.mdb hard link\r\n"); + fprintf(stderr, "errno %d: %s\r\n", errno, strerror(errno)); + return c3n; + } + if ( c3y == lok_o ) { // only link lock.mdb if it exists + if ( 0 < c3_link(luk_c, lok_c) ) { + fprintf(stderr, "disk: migrate: failed to create lock.mdb hard link\r\n"); + c3_rename(dat_c, dut_c); return c3n; } + } - // delete backup snapshot - c3_c bhk_c[8193], nop_c[8193], sop_c[8193]; - snprintf(bhk_c, sizeof(bhk_c), "%s/.urb/bhk", u3_Host.dir_c); - snprintf(nop_c, sizeof(nop_c), "%s/north.bin", bhk_c); - snprintf(sop_c, sizeof(sop_c), "%s/south.bin", bhk_c); - if ( c3n == c3_unlink(nop_c) ) { - fprintf(stderr, "disk: migrate: failed to delete bhk/north.bin\r\n"); - } else if ( c3n == c3_unlink(sop_c) ) { - fprintf(stderr, "disk: migrate: failed to delete bhk/south.bin\r\n"); - } else { - if ( c3n == c3_rmdir(bhk_c) ) { - fprintf(stderr, "disk: migrate: failed to delete bhk/\r\n"); - } - } + // rollover to new epoch + // log_u->mdb_u = old_u; + if ( c3n == u3_disk_epoc_init(log_u) ) { + fprintf(stderr, "roll: error: failed to initialize new epoch\r\n"); + return c3n; + } - // delete old lock.mdb and data.mdb files - if ( 0 != c3_unlink(luk_c) ) { - fprintf(stderr, "disk: migrate: failed to unlink lock.mdb\r\n"); - } - if ( 0 != c3_unlink(dut_c) ) { - fprintf(stderr, "disk: migrate: failed to unlink data.mdb\r\n"); - return c3n; // migration succeeds only if we can unlink data.mdb + // delete backup snapshot + c3_c bhk_c[8193], nop_c[8193], sop_c[8193]; + snprintf(bhk_c, sizeof(bhk_c), "%s/.urb/bhk", u3_Host.dir_c); + snprintf(nop_c, sizeof(nop_c), "%s/north.bin", bhk_c); + snprintf(sop_c, sizeof(sop_c), "%s/south.bin", bhk_c); + if ( c3n == c3_unlink(nop_c) ) { + fprintf(stderr, "disk: migrate: failed to delete bhk/north.bin\r\n"); + } else if ( c3n == c3_unlink(sop_c) ) { + fprintf(stderr, "disk: migrate: failed to delete bhk/south.bin\r\n"); + } else { + if ( c3n == c3_rmdir(bhk_c) ) { + fprintf(stderr, "disk: migrate: failed to delete bhk/\r\n"); } + } - // success - fprintf(stderr, "disk: migrated disk to v%d format\r\n", U3D_VER1); + // delete old lock.mdb and data.mdb files + if ( 0 != c3_unlink(luk_c) ) { + fprintf(stderr, "disk: migrate: failed to unlink lock.mdb\r\n"); } + if ( 0 != c3_unlink(dut_c) ) { + fprintf(stderr, "disk: migrate: failed to unlink data.mdb\r\n"); + return c3n; // migration succeeds only if we can unlink data.mdb + } + + // success + fprintf(stderr, "disk: migrated disk to v%d format\r\n", U3D_VER1); return c3y; } From d6fcb9875f800c8583241736a972f41ef580ef56 Mon Sep 17 00:00:00 2001 From: Matthew LeVan Date: Fri, 2 Jun 2023 10:36:17 -0400 Subject: [PATCH 037/271] disk: fix duplicate code --- pkg/vere/disk.c | 32 ++++++++++++++------------------ 1 file changed, 14 insertions(+), 18 deletions(-) diff --git a/pkg/vere/disk.c b/pkg/vere/disk.c index f0807bbedf..5046a3a1c6 100644 --- a/pkg/vere/disk.c +++ b/pkg/vere/disk.c @@ -1263,19 +1263,19 @@ c3_o u3_disk_migrate(u3_disk* log_u) */ // check if data.mdb is readable in log directory - c3_o dat_o = c3n; - c3_c dat_c[8193]; - snprintf(dat_c, sizeof(dat_c), "%s/data.mdb", log_u->com_u->pax_c); - if ( 0 == access(dat_c, R_OK) ) { - dat_o = c3y; + c3_o dut_o = c3n; + c3_c dut_c[8193]; + snprintf(dut_c, sizeof(dut_c), "%s/data.mdb", log_u->com_u->pax_c); + if ( 0 == access(dut_c, R_OK) ) { + dut_o = c3y; } // check if lock.mdb is readable in log directory - c3_o lok_o = c3n; - c3_c lok_c[8193]; - snprintf(lok_c, sizeof(dat_c), "%s/lock.mdb", log_u->com_u->pax_c); - if ( 0 == access(dat_c, R_OK) ) { - lok_o = c3y; + c3_o luk_o = c3n; + c3_c luk_c[8193]; + snprintf(luk_c, sizeof(luk_c), "%s/lock.mdb", log_u->com_u->pax_c); + if ( 0 == access(luk_c, R_OK) ) { + luk_o = c3y; } // if fresh boot, initialize disk v1 @@ -1291,8 +1291,8 @@ c3_o u3_disk_migrate(u3_disk* log_u) return c3y; } - // if data.mdb is not readable, skip migration - if ( c3n == dat_o ) return c3y; + // if .urb/log/data.mdb is not readable, skip migration + if ( c3n == dut_o ) return c3y; // migrate existing pier which has either: // - not started the migration, or @@ -1329,11 +1329,7 @@ c3_o u3_disk_migrate(u3_disk* log_u) } // create hard links to data.mdb and lock.mdb in 0i0/ - c3_c dut_c[8193], luk_c[8193]; // old paths - snprintf(dut_c, sizeof(dut_c), "%s/data.mdb", log_u->com_u->pax_c); - snprintf(luk_c, sizeof(luk_c), "%s/lock.mdb", log_u->com_u->pax_c); - - c3_c epo_c[8193]; + c3_c epo_c[8193], dat_c[8193], lok_c[8193]; snprintf(epo_c, sizeof(epo_c), "%s/0i0", log_u->com_u->pax_c); snprintf(dat_c, sizeof(dat_c), "%s/data.mdb", epo_c); snprintf(lok_c, sizeof(lok_c), "%s/lock.mdb", epo_c); @@ -1343,7 +1339,7 @@ c3_o u3_disk_migrate(u3_disk* log_u) fprintf(stderr, "errno %d: %s\r\n", errno, strerror(errno)); return c3n; } - if ( c3y == lok_o ) { // only link lock.mdb if it exists + if ( c3y == luk_o ) { // only link lock.mdb if it exists if ( 0 < c3_link(luk_c, lok_c) ) { fprintf(stderr, "disk: migrate: failed to create lock.mdb hard link\r\n"); c3_rename(dat_c, dut_c); From 259cda50e2a617fbdc27f75e188a27354c437837 Mon Sep 17 00:00:00 2001 From: Alex Shelkovnykov Date: Sun, 4 Jun 2023 13:38:11 +0000 Subject: [PATCH 038/271] jets: match hoon changes to bif:by --- pkg/noun/jets/d/by_bif.c | 6 ++---- pkg/noun/jets/d/by_dif.c | 5 +++-- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/pkg/noun/jets/d/by_bif.c b/pkg/noun/jets/d/by_bif.c index d9e27d1756..0a2f23a666 100644 --- a/pkg/noun/jets/d/by_bif.c +++ b/pkg/noun/jets/d/by_bif.c @@ -17,19 +17,17 @@ _b_bif_putroot(u3_noun a, else { u3_noun n_a, l_a, r_a; u3_noun p_n_a, q_n_a; - u3_noun p_b, q_b; u3x_trel(a, &n_a, &l_a, &r_a); - u3x_cell(b, &p_b, &q_b); u3x_cell(n_a, &p_n_a, &q_n_a); - if ( c3y == u3r_sing(p_b, p_n_a) ) { + if ( c3y == u3r_sing(b, p_n_a) ) { return u3nt(u3k(b), u3k(l_a), u3k(r_a)); } else { u3_noun c, n_c, l_c, r_c; u3_noun d; - if ( c3y == u3qc_gor(p_b, p_n_a) ) { + if ( c3y == u3qc_gor(b, p_n_a) ) { c = _b_bif_putroot(l_a, b); u3r_trel(c, &n_c, &l_c, &r_c); d = u3nt(u3k(n_c), diff --git a/pkg/noun/jets/d/by_dif.c b/pkg/noun/jets/d/by_dif.c index a3f35f3cde..44dcd1ad1b 100644 --- a/pkg/noun/jets/d/by_dif.c +++ b/pkg/noun/jets/d/by_dif.c @@ -61,11 +61,12 @@ u3qdb_dif(u3_noun a, return u3k(a); } else { - u3_noun n_b, l_b, r_b; + u3_noun n_b, p_n_b, q_n_b, l_b, r_b; u3_noun c, l_c, r_c; u3x_trel(b, &n_b, &l_b, &r_b); + u3x_cell(n_b, &p_n_b, &q_n_b); - c = u3qdb_bif(a, n_b); + c = u3qdb_bif(a, p_n_b); u3x_cell(c, &l_c, &r_c); u3_noun d = u3qdb_dif(l_c, l_b); From 1886844f4abbd4aac66cc91ec80c661d256c1817 Mon Sep 17 00:00:00 2001 From: barter-simsum Date: Fri, 2 Jun 2023 10:48:33 -0400 Subject: [PATCH 039/271] remove redundant dut_o in u3_disk_migrate --- pkg/vere/disk.c | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/pkg/vere/disk.c b/pkg/vere/disk.c index 5046a3a1c6..81eb3a600d 100644 --- a/pkg/vere/disk.c +++ b/pkg/vere/disk.c @@ -1263,11 +1263,12 @@ c3_o u3_disk_migrate(u3_disk* log_u) */ // check if data.mdb is readable in log directory - c3_o dut_o = c3n; c3_c dut_c[8193]; snprintf(dut_c, sizeof(dut_c), "%s/data.mdb", log_u->com_u->pax_c); - if ( 0 == access(dut_c, R_OK) ) { - dut_o = c3y; + if ( !_(u3_Host.ops_u.nuu) + && 0 != access(dut_c, R_OK) ) { + // if .urb/log/data.mdb is not readable, skip migration + return c3y; } // check if lock.mdb is readable in log directory @@ -1290,9 +1291,6 @@ c3_o u3_disk_migrate(u3_disk* log_u) return c3y; } - - // if .urb/log/data.mdb is not readable, skip migration - if ( c3n == dut_o ) return c3y; // migrate existing pier which has either: // - not started the migration, or From d59b0af5497add0f3c36a72d4deb86d138e0453a Mon Sep 17 00:00:00 2001 From: Matthew LeVan Date: Mon, 5 Jun 2023 11:05:40 -0400 Subject: [PATCH 040/271] disk: make `u3_disk_epoc_vere` safer --- pkg/vere/disk.c | 33 +++++++++++++++++++++++---------- 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/pkg/vere/disk.c b/pkg/vere/disk.c index 81eb3a600d..eb1b25792c 100644 --- a/pkg/vere/disk.c +++ b/pkg/vere/disk.c @@ -1216,25 +1216,38 @@ u3_disk_epoc_last(u3_disk* log_u, c3_d* lat_d) c3_o u3_disk_epoc_vere(u3_disk* log_u, c3_d epo_d, c3_c* ver_w) { - // XX check implementation (similar code is in king.c, _king_get_pace) - // should probably write a generic function to read a string from - // a file into malloc'd memory, with an optional whitespace trimming - c3_c ver_c[8193]; - snprintf(ver_c, sizeof(ver_c), "%s/0i%" PRIc3_d "/vere.txt", - log_u->com_u->pax_c, epo_d); - - FILE* fil_u = fopen(ver_c, "r"); - if ( NULL == fil_u ) { + struct stat buf_u; + c3_c* ver_c; + c3_w red_w, len_w; + c3_i ret_i, fid_i; + ret_i = asprintf(&ver_c, "%s/0i%" PRIc3_d "/vere.txt", + log_u->com_u->pax_c, epo_d); + u3_assert( ret_i > 0 ); + + fid_i = c3_open(ver_c, O_RDONLY, 0644); + + if ( (fid_i < 0) || (fstat(fid_i, &buf_u) < 0) ) { fprintf(stderr, "disk: failed to open vere.txt in epoch 0i%" PRIc3_d "\r\n", epo_d); return c3n; } - if ( 1 != fscanf(fil_u, "%s", ver_w) ) { + len_w = buf_u.st_size; + red_w = read(fid_i, ver_w, len_w); + close(fid_i); + + if ( len_w != red_w ) { fprintf(stderr, "disk: failed to read vere.txt in epoch 0i%" PRIc3_d "\r\n", epo_d); return c3n; } + + // trim trailing whitespace + ver_w[len_w] = 0; + while ( len_w-- && isspace(ver_w[len_w]) ) { + ver_w[len_w] = 0; + } + return c3y; } From c50906de6e69cf11a94ed9df245b51371783858b Mon Sep 17 00:00:00 2001 From: Matthew LeVan Date: Mon, 5 Jun 2023 11:29:17 -0400 Subject: [PATCH 041/271] disk: cleanup `#include` directives --- pkg/noun/noun.h | 1 - pkg/noun/urth.c | 1 - pkg/vere/disk.c | 1 + 3 files changed, 1 insertion(+), 2 deletions(-) diff --git a/pkg/noun/noun.h b/pkg/noun/noun.h index b493307288..965330a519 100644 --- a/pkg/noun/noun.h +++ b/pkg/noun/noun.h @@ -15,7 +15,6 @@ #include "types.h" #include "vortex.h" #include "zave.h" -#include "events.h" #include "imprison.h" #include "log.h" #include "nock.h" diff --git a/pkg/noun/urth.c b/pkg/noun/urth.c index a4a52e2522..05ca4db33c 100644 --- a/pkg/noun/urth.c +++ b/pkg/noun/urth.c @@ -8,7 +8,6 @@ #include #include "allocate.h" -#include "events.h" #include "hashtable.h" #include "imprison.h" #include "jets.h" diff --git a/pkg/vere/disk.c b/pkg/vere/disk.c index eb1b25792c..23be66ba4b 100644 --- a/pkg/vere/disk.c +++ b/pkg/vere/disk.c @@ -1,6 +1,7 @@ /// @file #include "noun.h" +#include "events.h" #include "vere.h" #include "version.h" #include "db/lmdb.h" From 611856662e9624bbe3f0a4f3466af209d44c87fc Mon Sep 17 00:00:00 2001 From: Matthew LeVan Date: Mon, 5 Jun 2023 11:34:53 -0400 Subject: [PATCH 042/271] lmdb: restore `siz_i` parameter in `u3_lmdb_init` --- pkg/vere/db/lmdb.c | 11 +---------- pkg/vere/db/lmdb.h | 2 +- pkg/vere/disk.c | 36 ++++++++++++++++++++++++++++++++---- 3 files changed, 34 insertions(+), 15 deletions(-) diff --git a/pkg/vere/db/lmdb.c b/pkg/vere/db/lmdb.c index d9fed4c0de..8f8864b77f 100644 --- a/pkg/vere/db/lmdb.c +++ b/pkg/vere/db/lmdb.c @@ -38,20 +38,11 @@ intmax_t mdb_get_filesize(mdb_filehandle_t han_u); /* u3_lmdb_init(): open lmdb at [pax_c], mmap up to [siz_i]. */ MDB_env* -u3_lmdb_init(const c3_c* pax_c) +u3_lmdb_init(const c3_c* pax_c, size_t siz_i) { MDB_env* env_u; c3_w ret_w; - // calculate db size based on architecture - const size_t siz_i = - // 500 GiB is as large as musl on aarch64 wants to allow - #if (defined(U3_CPU_aarch64) && defined(U3_OS_linux)) - 0x7d00000000; - #else - 0x10000000000; - #endif - if ( (ret_w = mdb_env_create(&env_u)) ) { mdb_logerror(stderr, ret_w, "lmdb: init fail"); return 0; diff --git a/pkg/vere/db/lmdb.h b/pkg/vere/db/lmdb.h index 7aa996001f..aab4e6c9f7 100644 --- a/pkg/vere/db/lmdb.h +++ b/pkg/vere/db/lmdb.h @@ -22,7 +22,7 @@ /* u3_lmdb_init(): open lmdb at [pax_c], mmap up to [siz_i]. */ MDB_env* - u3_lmdb_init(const c3_c* pax_c); + u3_lmdb_init(const c3_c* pax_c, size_t siz_i); /* u3_lmdb_exit(): close lmdb. */ diff --git a/pkg/vere/disk.c b/pkg/vere/disk.c index 23be66ba4b..d34c5b852a 100644 --- a/pkg/vere/disk.c +++ b/pkg/vere/disk.c @@ -909,7 +909,14 @@ u3_disk_init(c3_c* pax_c, u3_disk_cb cb_u) snprintf(epo_c, 8192, "%s/0i%" PRIc3_d, log_c, lat_d); // initialize latest epoch's db - if ( 0 == (log_u->mdb_u = u3_lmdb_init(epo_c)) ) { + const size_t siz_i = + // 500 GiB is as large as musl on aarch64 wants to allow + #if (defined(U3_CPU_aarch64) && defined(U3_OS_linux)) + 0x7d00000000; + #else + 0x10000000000; + #endif + if ( 0 == (log_u->mdb_u = u3_lmdb_init(epo_c, siz_i)) ) { fprintf(stderr, "disk: failed to initialize database\r\n"); c3_free(log_u); return 0; @@ -998,7 +1005,14 @@ u3_disk_epoc_good(u3_disk* log_u, c3_d epo_d) dat_o = c3y; // check if we can open data.mdb MDB_env* env_u; - if ( 0 != (env_u = u3_lmdb_init(epo_c)) ) { + const size_t siz_i = + // 500 GiB is as large as musl on aarch64 wants to allow + #if (defined(U3_CPU_aarch64) && defined(U3_OS_linux)) + 0x7d00000000; + #else + 0x10000000000; + #endif + if ( 0 != (env_u = u3_lmdb_init(epo_c, siz_i)) ) { mdb_o = c3y; } @@ -1126,7 +1140,14 @@ u3_disk_epoc_init(u3_disk* log_u) if ( c3y == u3_Host.ops_u.nuu || new_d > 0 ) { c3_c dat_c[8193]; snprintf(dat_c, sizeof(dat_c), "%s/data.mdb", epo_c); - if ( 0 == (log_u->mdb_u = u3_lmdb_init(epo_c)) ) { + const size_t siz_i = + // 500 GiB is as large as musl on aarch64 wants to allow + #if (defined(U3_CPU_aarch64) && defined(U3_OS_linux)) + 0x7d00000000; + #else + 0x10000000000; + #endif + if ( 0 == (log_u->mdb_u = u3_lmdb_init(epo_c, siz_i)) ) { fprintf(stderr, "disk: failed to initialize database\r\n"); c3_free(log_u); goto fail; @@ -1312,7 +1333,14 @@ c3_o u3_disk_migrate(u3_disk* log_u) // initialize pre-migrated lmdb { - if ( 0 == (log_u->mdb_u = u3_lmdb_init(log_u->com_u->pax_c)) ) { + const size_t siz_i = + // 500 GiB is as large as musl on aarch64 wants to allow + #if (defined(U3_CPU_aarch64) && defined(U3_OS_linux)) + 0x7d00000000; + #else + 0x10000000000; + #endif + if ( 0 == (log_u->mdb_u = u3_lmdb_init(log_u->com_u->pax_c, siz_i)) ) { fprintf(stderr, "disk: failed to initialize database\r\n"); return c3n; } From d5aff59e419ff5f4674eb7c6471dc5d2d2924218 Mon Sep 17 00:00:00 2001 From: Matthew LeVan Date: Tue, 6 Jun 2023 14:29:37 -0400 Subject: [PATCH 043/271] disk: define `siz_i` as a global, static constant --- pkg/vere/disk.c | 39 +++++++++------------------------------ 1 file changed, 9 insertions(+), 30 deletions(-) diff --git a/pkg/vere/disk.c b/pkg/vere/disk.c index d34c5b852a..7ad6420a6c 100644 --- a/pkg/vere/disk.c +++ b/pkg/vere/disk.c @@ -24,6 +24,15 @@ struct _cd_save { struct _u3_disk* log_u; }; +// for u3_lmdb_init() calls +static const size_t siz_i = +#if (defined(U3_CPU_aarch64) && defined(U3_OS_linux)) + // 500 GiB is as large as musl on aarch64 wants to allow + 0x7d00000000; +#else + 0x10000000000; +#endif + #undef VERBOSE_DISK #undef DISK_TRACE_JAM #undef DISK_TRACE_CUE @@ -909,13 +918,6 @@ u3_disk_init(c3_c* pax_c, u3_disk_cb cb_u) snprintf(epo_c, 8192, "%s/0i%" PRIc3_d, log_c, lat_d); // initialize latest epoch's db - const size_t siz_i = - // 500 GiB is as large as musl on aarch64 wants to allow - #if (defined(U3_CPU_aarch64) && defined(U3_OS_linux)) - 0x7d00000000; - #else - 0x10000000000; - #endif if ( 0 == (log_u->mdb_u = u3_lmdb_init(epo_c, siz_i)) ) { fprintf(stderr, "disk: failed to initialize database\r\n"); c3_free(log_u); @@ -1005,13 +1007,6 @@ u3_disk_epoc_good(u3_disk* log_u, c3_d epo_d) dat_o = c3y; // check if we can open data.mdb MDB_env* env_u; - const size_t siz_i = - // 500 GiB is as large as musl on aarch64 wants to allow - #if (defined(U3_CPU_aarch64) && defined(U3_OS_linux)) - 0x7d00000000; - #else - 0x10000000000; - #endif if ( 0 != (env_u = u3_lmdb_init(epo_c, siz_i)) ) { mdb_o = c3y; } @@ -1140,13 +1135,6 @@ u3_disk_epoc_init(u3_disk* log_u) if ( c3y == u3_Host.ops_u.nuu || new_d > 0 ) { c3_c dat_c[8193]; snprintf(dat_c, sizeof(dat_c), "%s/data.mdb", epo_c); - const size_t siz_i = - // 500 GiB is as large as musl on aarch64 wants to allow - #if (defined(U3_CPU_aarch64) && defined(U3_OS_linux)) - 0x7d00000000; - #else - 0x10000000000; - #endif if ( 0 == (log_u->mdb_u = u3_lmdb_init(epo_c, siz_i)) ) { fprintf(stderr, "disk: failed to initialize database\r\n"); c3_free(log_u); @@ -1333,13 +1321,6 @@ c3_o u3_disk_migrate(u3_disk* log_u) // initialize pre-migrated lmdb { - const size_t siz_i = - // 500 GiB is as large as musl on aarch64 wants to allow - #if (defined(U3_CPU_aarch64) && defined(U3_OS_linux)) - 0x7d00000000; - #else - 0x10000000000; - #endif if ( 0 == (log_u->mdb_u = u3_lmdb_init(log_u->com_u->pax_c, siz_i)) ) { fprintf(stderr, "disk: failed to initialize database\r\n"); return c3n; @@ -1382,13 +1363,11 @@ c3_o u3_disk_migrate(u3_disk* log_u) if ( c3y == luk_o ) { // only link lock.mdb if it exists if ( 0 < c3_link(luk_c, lok_c) ) { fprintf(stderr, "disk: migrate: failed to create lock.mdb hard link\r\n"); - c3_rename(dat_c, dut_c); return c3n; } } // rollover to new epoch - // log_u->mdb_u = old_u; if ( c3n == u3_disk_epoc_init(log_u) ) { fprintf(stderr, "roll: error: failed to initialize new epoch\r\n"); return c3n; From 563a0c24a48870db0b08e19a11dcce7c51a9b8bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miko=C5=82aj=20Paraniak?= Date: Thu, 8 Jun 2023 10:00:50 +0800 Subject: [PATCH 044/271] jets: fixes indentation in +fitz jet --- pkg/noun/jets/f/fitz.c | 90 +++++++++++++++++++++--------------------- 1 file changed, 45 insertions(+), 45 deletions(-) diff --git a/pkg/noun/jets/f/fitz.c b/pkg/noun/jets/f/fitz.c index f8cff2b1c9..a08fd91871 100644 --- a/pkg/noun/jets/f/fitz.c +++ b/pkg/noun/jets/f/fitz.c @@ -6,62 +6,62 @@ #include "noun.h" - static u3_noun - _fitz_fiz(u3_noun yaz, - u3_noun wix) - { - c3_w yaz_w = u3r_met(3, yaz); - c3_w wix_w = u3r_met(3, wix); - c3_y yaz_y, wix_y; +static u3_noun +_fitz_fiz(u3_noun yaz, + u3_noun wix) +{ + c3_w yaz_w = u3r_met(3, yaz); + c3_w wix_w = u3r_met(3, wix); + c3_y yaz_y, wix_y; - yaz_y = (0 == yaz_w) ? 0 : u3r_byte((yaz_w - 1), yaz); - if ( (yaz_y < 'A') || (yaz_y > 'Z') ) yaz_y = 0; + yaz_y = (0 == yaz_w) ? 0 : u3r_byte((yaz_w - 1), yaz); + if ( (yaz_y < 'A') || (yaz_y > 'Z') ) yaz_y = 0; - wix_y = (0 == wix_w) ? 0 : u3r_byte((wix_w - 1), wix); - if ( (wix_y < 'A') || (wix_y > 'Z') ) wix_y = 0; + wix_y = (0 == wix_w) ? 0 : u3r_byte((wix_w - 1), wix); + if ( (wix_y < 'A') || (wix_y > 'Z') ) wix_y = 0; - if ( yaz_y && wix_y ) { - if ( !wix_y || (wix_y > yaz_y) ) { - return c3n; - } + if ( yaz_y && wix_y ) { + if ( !wix_y || (wix_y > yaz_y) ) { + return c3n; } - return c3y; } + return c3y; +} - u3_noun - u3qf_fitz(u3_noun yaz, - u3_noun wix) - { - c3_w i_w, met_w = c3_min(u3r_met(3, yaz), u3r_met(3, wix)); +u3_noun +u3qf_fitz(u3_noun yaz, + u3_noun wix) +{ + c3_w i_w, met_w = c3_min(u3r_met(3, yaz), u3r_met(3, wix)); - if ( c3n == _fitz_fiz(yaz, wix) ) { - return c3n; - } - for ( i_w = 0; i_w < met_w; i_w++ ) { - c3_y yaz_y = u3r_byte(i_w, yaz); - c3_y wix_y = u3r_byte(i_w, wix); + if ( c3n == _fitz_fiz(yaz, wix) ) { + return c3n; + } + for ( i_w = 0; i_w < met_w; i_w++ ) { + c3_y yaz_y = u3r_byte(i_w, yaz); + c3_y wix_y = u3r_byte(i_w, wix); - if ( (yaz_y >= 'A') && (yaz_y <= 'Z') ) yaz_y = 0; - if ( (wix_y >= 'A') && (wix_y <= 'Z') ) wix_y = 0; + if ( (yaz_y >= 'A') && (yaz_y <= 'Z') ) yaz_y = 0; + if ( (wix_y >= 'A') && (wix_y <= 'Z') ) wix_y = 0; - if ( yaz_y && wix_y && (yaz_y != wix_y) ) { - return c3n; - } + if ( yaz_y && wix_y && (yaz_y != wix_y) ) { + return c3n; } - return c3y; } + return c3y; +} - u3_noun - u3wf_fitz(u3_noun cor) - { - u3_noun yaz, wix; +u3_noun +u3wf_fitz(u3_noun cor) +{ + u3_noun yaz, wix; - if ( (c3n == u3r_mean(cor, u3x_sam_2, &yaz, u3x_sam_3, &wix, 0)) || - (c3n == u3ud(yaz)) || - (c3n == u3ud(wix)) ) - { - return u3m_bail(c3__fail); - } else { - return u3qf_fitz(yaz, wix); - } + if ( (c3n == u3r_mean(cor, u3x_sam_2, &yaz, u3x_sam_3, &wix, 0)) || + (c3n == u3ud(yaz)) || + (c3n == u3ud(wix)) ) + { + return u3m_bail(c3__fail); + } else { + return u3qf_fitz(yaz, wix); } +} From 88cc266f5710a6690c42c35366f313e0cb216908 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miko=C5=82aj=20Paraniak?= Date: Thu, 8 Jun 2023 10:41:44 +0800 Subject: [PATCH 045/271] jets: fix a jet mismatch in +fitz The +fitz jet erronously excludes all uppercase letters but the last one from aura compatibility check. In +fitz aura compatibility check, only the last uppercase letter is considered a bit-width annotation. --- pkg/noun/jets/f/fitz.c | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/pkg/noun/jets/f/fitz.c b/pkg/noun/jets/f/fitz.c index a08fd91871..b1c2a5dfb3 100644 --- a/pkg/noun/jets/f/fitz.c +++ b/pkg/noun/jets/f/fitz.c @@ -5,7 +5,6 @@ #include "noun.h" - static u3_noun _fitz_fiz(u3_noun yaz, u3_noun wix) @@ -21,10 +20,11 @@ _fitz_fiz(u3_noun yaz, if ( (wix_y < 'A') || (wix_y > 'Z') ) wix_y = 0; if ( yaz_y && wix_y ) { - if ( !wix_y || (wix_y > yaz_y) ) { + if ( wix_y > yaz_y ) { return c3n; } } + return c3y; } @@ -32,7 +32,10 @@ u3_noun u3qf_fitz(u3_noun yaz, u3_noun wix) { - c3_w i_w, met_w = c3_min(u3r_met(3, yaz), u3r_met(3, wix)); + c3_w yet_w = u3r_met(3, yaz); + c3_w wet_w = u3r_met(3, wix); + + c3_w i_w, met_w = c3_min(yet_w, wet_w); if ( c3n == _fitz_fiz(yaz, wix) ) { return c3n; @@ -41,10 +44,15 @@ u3qf_fitz(u3_noun yaz, c3_y yaz_y = u3r_byte(i_w, yaz); c3_y wix_y = u3r_byte(i_w, wix); - if ( (yaz_y >= 'A') && (yaz_y <= 'Z') ) yaz_y = 0; - if ( (wix_y >= 'A') && (wix_y <= 'Z') ) wix_y = 0; + if ( (i_w == (yet_w - 1)) && (yaz_y >= 'A') && (yaz_y <= 'Z')) { + return c3y; + } + + if ( (i_w == (wet_w - 1)) && (wix_y >= 'A') && (wix_y <= 'Z')) { + return c3y; + } - if ( yaz_y && wix_y && (yaz_y != wix_y) ) { + if ( yaz_y != wix_y ) { return c3n; } } From cc2557cd8e11018d38d319afebdf6c8df0fed8f9 Mon Sep 17 00:00:00 2001 From: Matthew LeVan Date: Fri, 9 Jun 2023 16:09:34 -0400 Subject: [PATCH 046/271] disk: pare down `u3_disk_epoc_good` --- pkg/vere/disk.c | 104 +++++++++++++----------------------------------- 1 file changed, 27 insertions(+), 77 deletions(-) diff --git a/pkg/vere/disk.c b/pkg/vere/disk.c index 7ad6420a6c..9459fecedc 100644 --- a/pkg/vere/disk.c +++ b/pkg/vere/disk.c @@ -5,6 +5,7 @@ #include "vere.h" #include "version.h" #include "db/lmdb.h" +#include struct _cd_read { uv_timer_t tim_u; @@ -905,14 +906,6 @@ u3_disk_init(c3_c* pax_c, u3_disk_cb cb_u) return 0; } - // get binary version from latest epoch - c3_c ver_c[8193]; - if ( c3n == u3_disk_epoc_vere(log_u, lat_d, ver_c) ) { - fprintf(stderr, "disk: failed to load epoch version\r\n"); - c3_free(log_u); - return 0; - } - // set path to latest epoch c3_c epo_c[8193]; snprintf(epo_c, 8192, "%s/0i%" PRIc3_d, log_c, lat_d); @@ -946,6 +939,12 @@ u3_disk_init(c3_c* pax_c, u3_disk_cb cb_u) // if binary version of latest epoch is not the same as the // running binary, then we need to create a new epoch + c3_c ver_c[8193]; + if ( c3n == u3_disk_epoc_vere(log_u, lat_d, ver_c) ) { + fprintf(stderr, "disk: failed to load epoch version\r\n"); + c3_free(log_u); + return 0; + } if ( 0 != strcmp(ver_c, URBIT_VERSION) ) { if ( c3n == u3_disk_epoc_init(log_u) ) { fprintf(stderr, "disk: failed to initialize epoch\r\n"); @@ -971,14 +970,10 @@ u3_disk_init(c3_c* pax_c, u3_disk_cb cb_u) c3_o u3_disk_epoc_good(u3_disk* log_u, c3_d epo_d) { - /* a "valid" epoch is currently defined as a writable folder in - * the /.urb/log directory named with a @ui and contains: - * - a writable data.mdb file which can be opened by lmdb and contains 1+ events - * - a writable epoc.txt file - * - a writable vere.txt file - * - * note that this does not check if the epoch even contains snapshot files, - * let alone whether they're valid or not + /* this function is used to check whether or not we can overwrite + * an epoch. if the epoch contains an openable data.mdb file that + * also contains events, then we consider it good and *cannot* + * overwrite it. if not, then we consider it bad and *can* overwrite it. */ c3_o ret_o = c3y; // return code @@ -987,78 +982,33 @@ u3_disk_epoc_good(u3_disk* log_u, c3_d epo_d) c3_c epo_c[8193], dat_c[8193], epv_c[8193], biv_c[8193]; snprintf(epo_c, sizeof(epo_c), "%s/0i%" PRIc3_d, log_u->com_u->pax_c, epo_d); snprintf(dat_c, sizeof(dat_c), "%s/data.mdb", epo_c); - snprintf(epv_c, sizeof(epv_c), "%s/epoc.txt", epo_c); - snprintf(biv_c, sizeof(biv_c), "%s/vere.txt", epo_c); - c3_o dir_o = c3n; // directory is writable - c3_o dat_o = c3n; // data.mdb is writable - c3_o len_o = c3n; // data.mdb has events - c3_o mdb_o = c3n; // data.mdb can be opened - c3_o epv_o = c3n; // epoc.txt is writable - c3_o biv_o = c3n; // vere.txt is writable - - // check if dir is writable - if ( 0 == access(epo_c, W_OK) ) { - dir_o = c3y; - } - - // check if data.mdb is writable - if ( 0 == access(dat_c, W_OK) ) { - dat_o = c3y; - // check if we can open data.mdb - MDB_env* env_u; - if ( 0 != (env_u = u3_lmdb_init(epo_c, siz_i)) ) { - mdb_o = c3y; - } + c3_o len_o = c3n; // data.mdb is empty - // check if there are any events in the database - c3_d low_d, hig_d; - if ( 0 != u3_lmdb_gulf(env_u, &low_d, &hig_d) ) { - fprintf(stderr, "disk: failed to get first/last event numbers\r\n"); - ret_o = c3n; - } else { - if ( low_d == hig_d == 0 ) { - len_o = c3y; // yes, data.mdb has events - } - } - u3_lmdb_exit(env_u); - } - - // check if epoc.txt is writable - if ( 0 == access(epv_c, W_OK) ) { - epv_o = c3y; + // check if we can open data.mdb + MDB_env* env_u; + if ( 0 == (env_u = u3_lmdb_init(epo_c, siz_i)) ) { + fprintf(stderr, "disk: failed to open epoch 0i%" PRIc3_d "/data.mdb\r\n", epo_d); + exit(1); } - // check if vere.txt is writable - if ( 0 == access(biv_c, W_OK) ) { - biv_o = c3y; + // check if there are any events in the database + c3_d low_d, hig_d; + if ( 0 != u3_lmdb_gulf(env_u, &low_d, &hig_d) ) { + fprintf(stderr, "disk: failed to get first/last event numbers\r\n"); + ret_o = c3n; + } else { + if ( low_d == hig_d == 0 ) { + len_o = c3y; // yes, data.mdb is empty + } } + u3_lmdb_exit(env_u); // print error messages - if ( c3n == dir_o ) { - fprintf(stderr, "disk: epoch 0i%" PRIc3_d " is not writable\r\n", epo_d); - ret_o = c3n; - } - if ( c3n == dat_o ) { - fprintf(stderr, "disk: epoch 0i%" PRIc3_d "/data.mdb is not writable\r\n", epo_d); - ret_o = c3n; - } - if ( c3n == mdb_o ) { - fprintf(stderr, "disk: epoch 0i%" PRIc3_d "/data.mdb can't be opened\r\n", epo_d); - ret_o = c3n; - } if ( c3n == len_o ) { fprintf(stderr, "disk: epoch 0i%" PRIc3_d "/data.mdb has no events\r\n", epo_d); ret_o = c3n; } - if ( c3n == epv_o ) { - fprintf(stderr, "disk: epoch 0i%" PRIc3_d "/epoc.txt is not writable\r\n", epo_d); - ret_o = c3n; - } - if ( c3n == biv_o ) { - fprintf(stderr, "disk: epoch 0i%" PRIc3_d "/vere.txt is not writable\r\n", epo_d); - ret_o = c3n; - } return ret_o; } From 8703f11fd12df5883e5b24f05b7dee6e2103aa96 Mon Sep 17 00:00:00 2001 From: Matthew LeVan Date: Mon, 12 Jun 2023 10:23:28 -0400 Subject: [PATCH 047/271] disk: remove `u3_disk_epoc_good` and add argument to `u3_disk_epoc_init` --- pkg/vere/disk.c | 114 ++++++++++-------------------------------------- pkg/vere/main.c | 4 +- pkg/vere/vere.h | 7 +-- 3 files changed, 27 insertions(+), 98 deletions(-) diff --git a/pkg/vere/disk.c b/pkg/vere/disk.c index 9459fecedc..41da47653d 100644 --- a/pkg/vere/disk.c +++ b/pkg/vere/disk.c @@ -946,7 +946,7 @@ u3_disk_init(c3_c* pax_c, u3_disk_cb cb_u) return 0; } if ( 0 != strcmp(ver_c, URBIT_VERSION) ) { - if ( c3n == u3_disk_epoc_init(log_u) ) { + if ( c3n == u3_disk_epoc_init(log_u, log_u->dun_d) ) { fprintf(stderr, "disk: failed to initialize epoch\r\n"); c3_free(log_u); return 0; @@ -965,86 +965,21 @@ u3_disk_init(c3_c* pax_c, u3_disk_cb cb_u) return log_u; } -/* u3_disk_epoc_good: check for valid epoch. -*/ -c3_o -u3_disk_epoc_good(u3_disk* log_u, c3_d epo_d) -{ - /* this function is used to check whether or not we can overwrite - * an epoch. if the epoch contains an openable data.mdb file that - * also contains events, then we consider it good and *cannot* - * overwrite it. if not, then we consider it bad and *can* overwrite it. - */ - - c3_o ret_o = c3y; // return code - - // file paths - c3_c epo_c[8193], dat_c[8193], epv_c[8193], biv_c[8193]; - snprintf(epo_c, sizeof(epo_c), "%s/0i%" PRIc3_d, log_u->com_u->pax_c, epo_d); - snprintf(dat_c, sizeof(dat_c), "%s/data.mdb", epo_c); - - c3_o len_o = c3n; // data.mdb is empty - - // check if we can open data.mdb - MDB_env* env_u; - if ( 0 == (env_u = u3_lmdb_init(epo_c, siz_i)) ) { - fprintf(stderr, "disk: failed to open epoch 0i%" PRIc3_d "/data.mdb\r\n", epo_d); - exit(1); - } - - // check if there are any events in the database - c3_d low_d, hig_d; - if ( 0 != u3_lmdb_gulf(env_u, &low_d, &hig_d) ) { - fprintf(stderr, "disk: failed to get first/last event numbers\r\n"); - ret_o = c3n; - } else { - if ( low_d == hig_d == 0 ) { - len_o = c3y; // yes, data.mdb is empty - } - } - u3_lmdb_exit(env_u); - - // print error messages - if ( c3n == len_o ) { - fprintf(stderr, "disk: epoch 0i%" PRIc3_d "/data.mdb has no events\r\n", epo_d); - ret_o = c3n; - } - - return ret_o; -} - /* u3_disk_epoc_init: create new epoch. */ c3_o -u3_disk_epoc_init(u3_disk* log_u) +u3_disk_epoc_init(u3_disk* log_u, c3_d epo_d) { - // set new epoch number - c3_d lat_d, new_d; + // check if any epoch directories exist + c3_d lat_d; c3_o eps_o = u3_disk_epoc_last(log_u, &lat_d); - if ( c3n == eps_o ) { - // no epochs yet, so create first one 0i0 - new_d = 0; - } else if ( c3n == u3_disk_epoc_good(log_u, lat_d) ) { - // last epoch is invalid, so overwrite it - new_d = lat_d; - } else { - // last epoch is valid, so create next one - - // get first/last event numbers - c3_d fir_d, las_d; - if ( c3n == u3_lmdb_gulf(log_u->mdb_u, &fir_d, &las_d) ) { - fprintf(stderr, "disk: failed to get first/last event numbers\r\n"); - return c3n; - } - new_d = las_d; // create next epoch - } // create new epoch directory if it doesn't exist c3_c epo_c[8193]; - snprintf(epo_c, sizeof(epo_c), "%s/0i%" PRIc3_d, log_u->com_u->pax_c, new_d); + snprintf(epo_c, sizeof(epo_c), "%s/0i%" PRIc3_d, log_u->com_u->pax_c, epo_d); c3_d ret_d = c3_mkdir(epo_c, 0700); if ( ( ret_d < 0 ) && ( errno != EEXIST ) ) { - fprintf(stderr, "disk: failed to create epoch directory %" PRIc3_d "\r\n", new_d); + fprintf(stderr, "disk: failed to create epoch directory %" PRIc3_d "\r\n", epo_d); return c3n; } @@ -1063,7 +998,7 @@ u3_disk_epoc_init(u3_disk* log_u) fclose(biv_f); // copy snapshot (skip if first epoch) - if ( new_d > 0 ) { + if ( epo_d > 0 ) { if ( c3n == u3e_backup(epo_c, c3y) ) { fprintf(stderr, "disk: failed to copy snapshot to new epoch\r\n"); goto fail; @@ -1074,7 +1009,7 @@ u3_disk_epoc_init(u3_disk* log_u) c3_d who_d[2]; c3_o fak_o; c3_w lif_w; - if ( c3y == eps_o ) { + if ( c3y == eps_o ) { // skip if no epochs yet if ( c3y != u3_disk_read_meta(log_u->mdb_u, who_d, &fak_o, &lif_w) ) { fprintf(stderr, "disk: failed to read metadata\r\n"); goto fail; @@ -1082,7 +1017,7 @@ u3_disk_epoc_init(u3_disk* log_u) } // initialize db of new epoch - if ( c3y == u3_Host.ops_u.nuu || new_d > 0 ) { + if ( c3y == u3_Host.ops_u.nuu || epo_d > 0 ) { c3_c dat_c[8193]; snprintf(dat_c, sizeof(dat_c), "%s/data.mdb", epo_c); if ( 0 == (log_u->mdb_u = u3_lmdb_init(epo_c, siz_i)) ) { @@ -1153,7 +1088,7 @@ u3_disk_epoc_kill(u3_disk* log_u, c3_d epo_d) c3_o u3_disk_epoc_last(u3_disk* log_u, c3_d* lat_d) { - c3_o ret_o = c3n; // return no if no epoch directories exist + c3_o ret_o = c3n; // return c3n if no epoch directories exist *lat_d = 0; // initialize lat_d to 0 u3_dire* die_u = u3_foil_folder(log_u->com_u->pax_c); u3_dent* den_u = die_u->dil_u; @@ -1257,7 +1192,7 @@ c3_o u3_disk_migrate(u3_disk* log_u) fprintf(stderr, "disk: initializing disk with v%d format\r\n", U3D_VER1); // initialize first epoch "0i0" - if ( c3n == u3_disk_epoc_init(log_u) ) { + if ( c3n == u3_disk_epoc_init(log_u, 0) ) { fprintf(stderr, "disk: failed to initialize first epoch\r\n"); return c3n; } @@ -1286,16 +1221,16 @@ c3_o u3_disk_migrate(u3_disk* log_u) // ensure there's a current snapshot if ( u3_Host.eve_d != las_d ) { - fprintf(stderr, "disk: migrate: error: snapshot is out of date, please " + fprintf(stderr, "disk: snapshot is out of date, please " "start/shutdown your pier gracefully first\r\n"); - fprintf(stderr, "disk: migrate: eve_d (%" PRIc3_d ") != las_d (%" PRIc3_d ")\r\n", + fprintf(stderr, "disk: eve_d (%" PRIc3_d ") != las_d (%" PRIc3_d ")\r\n", u3_Host.eve_d, las_d); return c3n; } // initialize first epoch "0i0" - if ( c3n == u3_disk_epoc_init(log_u) ) { - fprintf(stderr, "disk: migrate: failed to initialize first epoch\r\n"); + if ( c3n == u3_disk_epoc_init(log_u, 0) ) { + fprintf(stderr, "disk: failed to initialize first epoch\r\n"); return c3n; } @@ -1306,20 +1241,19 @@ c3_o u3_disk_migrate(u3_disk* log_u) snprintf(lok_c, sizeof(lok_c), "%s/lock.mdb", epo_c); if ( 0 < c3_link(dut_c, dat_c) ) { - fprintf(stderr, "disk: migrate: failed to create data.mdb hard link\r\n"); - fprintf(stderr, "errno %d: %s\r\n", errno, strerror(errno)); + fprintf(stderr, "disk: failed to create data.mdb hard link\r\n"); return c3n; } if ( c3y == luk_o ) { // only link lock.mdb if it exists if ( 0 < c3_link(luk_c, lok_c) ) { - fprintf(stderr, "disk: migrate: failed to create lock.mdb hard link\r\n"); + fprintf(stderr, "disk: failed to create lock.mdb hard link\r\n"); return c3n; } } // rollover to new epoch - if ( c3n == u3_disk_epoc_init(log_u) ) { - fprintf(stderr, "roll: error: failed to initialize new epoch\r\n"); + if ( c3n == u3_disk_epoc_init(log_u, las_d) ) { + fprintf(stderr, "disk: failed to initialize new epoch\r\n"); return c3n; } @@ -1329,21 +1263,21 @@ c3_o u3_disk_migrate(u3_disk* log_u) snprintf(nop_c, sizeof(nop_c), "%s/north.bin", bhk_c); snprintf(sop_c, sizeof(sop_c), "%s/south.bin", bhk_c); if ( c3n == c3_unlink(nop_c) ) { - fprintf(stderr, "disk: migrate: failed to delete bhk/north.bin\r\n"); + fprintf(stderr, "disk: failed to delete bhk/north.bin\r\n"); } else if ( c3n == c3_unlink(sop_c) ) { - fprintf(stderr, "disk: migrate: failed to delete bhk/south.bin\r\n"); + fprintf(stderr, "disk: failed to delete bhk/south.bin\r\n"); } else { if ( c3n == c3_rmdir(bhk_c) ) { - fprintf(stderr, "disk: migrate: failed to delete bhk/\r\n"); + fprintf(stderr, "disk: failed to delete bhk/\r\n"); } } // delete old lock.mdb and data.mdb files if ( 0 != c3_unlink(luk_c) ) { - fprintf(stderr, "disk: migrate: failed to unlink lock.mdb\r\n"); + fprintf(stderr, "disk: failed to unlink lock.mdb\r\n"); } if ( 0 != c3_unlink(dut_c) ) { - fprintf(stderr, "disk: migrate: failed to unlink data.mdb\r\n"); + fprintf(stderr, "disk: failed to unlink data.mdb\r\n"); return c3n; // migration succeeds only if we can unlink data.mdb } diff --git a/pkg/vere/main.c b/pkg/vere/main.c index dc9d3819ef..d29d8fba20 100644 --- a/pkg/vere/main.c +++ b/pkg/vere/main.c @@ -2160,7 +2160,7 @@ _cw_chop(c3_i argc, c3_c* argv[]) } // create new epoch if latest isn't empty - if ( (fir_d != las_d) && (c3n == u3_disk_epoc_init(log_u)) ) { + if ( (fir_d != las_d) && (c3n == u3_disk_epoc_init(log_u, las_d)) ) { fprintf(stderr, "chop: failed to create new epoch\r\n"); exit(1); } @@ -2302,7 +2302,7 @@ _cw_roll(c3_i argc, c3_c* argv[]) fprintf(stderr, "roll: latest epoch already empty\r\n"); exit(0); } - else if ( c3n == u3_disk_epoc_init(log_u) ) { + else if ( c3n == u3_disk_epoc_init(log_u, las_d) ) { fprintf(stderr, "roll: failed to create new epoch\r\n"); exit(1); } diff --git a/pkg/vere/vere.h b/pkg/vere/vere.h index f3e47a98b6..8d613e77b8 100644 --- a/pkg/vere/vere.h +++ b/pkg/vere/vere.h @@ -984,15 +984,10 @@ void u3_disk_plan(u3_disk* log_u, u3_fact* tac_u); - /* u3_disk_epoc_good(): check for valid epoch. - */ - c3_o - u3_disk_epoc_good(u3_disk* log_u, c3_d epo_d); - /* u3_disk_epoc_init(): create new epoch. */ c3_o - u3_disk_epoc_init(u3_disk* log_u); + u3_disk_epoc_init(u3_disk* log_u, c3_d epo_d); /* u3_disk_epoc_kill(): delete an epoch. */ From 52f76bc93390122aa28711f6c44b15ae560264f2 Mon Sep 17 00:00:00 2001 From: Matthew LeVan Date: Thu, 15 Jun 2023 14:24:05 -0400 Subject: [PATCH 048/271] wip --- pkg/vere/main.c | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/pkg/vere/main.c b/pkg/vere/main.c index 0333d4d8f7..bb5b514bcc 100644 --- a/pkg/vere/main.c +++ b/pkg/vere/main.c @@ -2096,6 +2096,28 @@ _cw_play(c3_i argc, c3_c* argv[]) u3C.wag_w |= u3o_hashless; + // replaying events with the epoch system + // + // there should be one process that accounts for what is actually available + // in the pier and replays the least events possible to produce a valid, + // up-to-date snapshot. + // + // this replay process also uses the chk/ folder as a place to + // "accumulate" snapshots over the course of a replay if it needs to, + // overwriting the existing .bin files as it goes. + // + // 0. try loading the snapshot in chk/ + // a. if it is valid and up-to-date with the log + // i. if we executed via `urbit play`, exit 0 + // ii. if we executed via `urbit`, boot into live mode + // b. if it is invalid, try loading a snapshot from + // i. the snapshot in the latest epoch + // 1. if it is valid, replay the events and save the resulting + // snapshot into chk/, exit 0 or boot into live mode as before + // 2. if it is invalid, return an error that informs the user of + // his need for a serious surgical operation + // + // XX this should restore the epoch snapshot and replay that // if ( c3y == ful_o ) { From 20a3af06cf59581e1eafa2518db0c485f3b4589e Mon Sep 17 00:00:00 2001 From: Matthew LeVan Date: Tue, 20 Jun 2023 12:35:32 -0400 Subject: [PATCH 049/271] disk: set `epo_u` --- pkg/vere/disk.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/vere/disk.c b/pkg/vere/disk.c index 7371f5a3fa..b06ea35fec 100644 --- a/pkg/vere/disk.c +++ b/pkg/vere/disk.c @@ -1105,6 +1105,9 @@ u3_disk_init(c3_c* pax_c, u3_disk_cb cb_u) } } + // mark the latest epoch directory + log_u->epo_u = u3_foil_folder(epo_c); + // mark the log as live log_u->liv_o = c3y; } From 1af03ee813fe2068b6e68b56590a280dbe0fa3c2 Mon Sep 17 00:00:00 2001 From: Joe Bryan Date: Tue, 20 Jun 2023 18:37:39 -0400 Subject: [PATCH 050/271] vere: bumps hoon kelvin to %138 --- pkg/vere/pier.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/vere/pier.c b/pkg/vere/pier.c index 6f21269063..ebe4befd08 100644 --- a/pkg/vere/pier.c +++ b/pkg/vere/pier.c @@ -813,7 +813,7 @@ _pier_wyrd_card(u3_pier* pir_u) u3_noun kel = u3nl(u3nc(c3__zuse, VERE_ZUSE), // XX from both king and serf? u3nc(c3__lull, VERE_LULL), // XX from both king and serf? u3nc(c3__arvo, 237), // XX from both king and serf? - u3nc(c3__hoon, 139), // god_u->hon_y + u3nc(c3__hoon, 138), // god_u->hon_y u3nc(c3__nock, 4), // god_u->noc_y u3_none); u3_noun wir = u3nc(c3__arvo, u3_nul); From e5723adbe3ef4e08034eb7548ddcb5971a8bd2ba Mon Sep 17 00:00:00 2001 From: Matthew LeVan Date: Wed, 21 Jun 2023 11:58:42 -0400 Subject: [PATCH 051/271] disk: `urbit play` with epochs --- .gitignore | 4 +++ pkg/noun/events.c | 92 +++++++++++++++++++++++++++++++++-------------- pkg/noun/events.h | 5 +-- pkg/vere/disk.c | 6 ++-- pkg/vere/main.c | 19 +++++----- 5 files changed, 85 insertions(+), 41 deletions(-) diff --git a/.gitignore b/.gitignore index 251a0d7f0f..da5bf8e6c8 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,7 @@ +# IDEs. +/.vscode +/.idea + # Bazel. /.user.bazelrc /bazel-* diff --git a/pkg/noun/events.c b/pkg/noun/events.c index efb506f072..6da639e476 100644 --- a/pkg/noun/events.c +++ b/pkg/noun/events.c @@ -312,26 +312,44 @@ _ce_image_stat(u3e_image* img_u, c3_w* pgs_w) } } -/* _ce_image_open(): open or create image. +/* _ce_image_open(): open or create image, creating directories if necessary. */ static c3_o -_ce_image_open(u3e_image* img_u) +_ce_image_open(u3e_image* img_u, c3_c* ful_c) { c3_i mod_i = O_RDWR | O_CREAT; - c3_c ful_c[8193]; - snprintf(ful_c, 8192, "%s", u3P.dir_c); - c3_mkdir(ful_c, 0700); + // create directories if necessary + // + { + c3_c pax_c[8192]; + c3_c* end_c = pax_c + 8192; + c3_c* cur_c = pax_c + snprintf(pax_c, 8192, "%s", ful_c); - snprintf(ful_c, 8192, "%s/.urb", u3P.dir_c); - c3_mkdir(ful_c, 0700); + while ( cur_c < end_c ) { + c3_c* nxt_c = strchr(cur_c, '/'); - snprintf(ful_c, 8192, "%s/.urb/chk", u3P.dir_c); - c3_mkdir(ful_c, 0700); + if ( !nxt_c ) { + break; + } + else { + *nxt_c = 0; + + if ( 0 != mkdir(pax_c, 0777) && EEXIST != errno ) { + fprintf(stderr, "loom: mkdir %s: %s\r\n", pax_c, strerror(errno)); + return c3n; + } - snprintf(ful_c, 8192, "%s/.urb/chk/%s.bin", u3P.dir_c, img_u->nam_c); - if ( -1 == (img_u->fid_i = c3_open(ful_c, mod_i, 0666)) ) { - fprintf(stderr, "loom: c3_open %s: %s\r\n", ful_c, strerror(errno)); + *nxt_c = '/'; + cur_c = nxt_c + 1; + } + } + } + + c3_c pax_c[8192]; + snprintf(pax_c, 8192, "%s/%s.bin", ful_c, img_u->nam_c); + if ( -1 == (img_u->fid_i = c3_open(pax_c, mod_i, 0666)) ) { + fprintf(stderr, "loom: c3_open %s: %s\r\n", pax_c, strerror(errno)); return c3n; } else if ( c3n == _ce_image_stat(img_u, &img_u->pgs_w) ) { @@ -1231,17 +1249,24 @@ _ce_image_copy(u3e_image* fom_u, u3e_image* tou_u) return c3y; } -/* u3e_backup(): copy snapshot to pax_c, overwrite optionally. +/* u3e_backup(): copy snapshot from [pux_c] to [pax_c], + * overwriting optionally. note that image files must + * be named "north" and "south". */ c3_o -u3e_backup(c3_c* pax_c, c3_o ovw_o) +u3e_backup(c3_c* pux_c, c3_c* pax_c, c3_o ovw_o) { - u3e_image nop_u = { .nam_c = "north", .pgs_w = 0 }; - u3e_image sop_u = { .nam_c = "south", .pgs_w = 0 }; + // source image files from [pux_c] + u3e_image nux_u = { .nam_c = "north", .pgs_w = 0 }; + u3e_image sux_u = { .nam_c = "south", .pgs_w = 0 }; + + // destination image files to [pax_c] + u3e_image nax_u = { .nam_c = "north" }; + u3e_image sax_u = { .nam_c = "south" }; c3_i mod_i = O_RDWR | O_CREAT; - if ( !pax_c ) { + if ( !pux_c || !pax_c ) { fprintf(stderr, "loom: image backup: bad path\r\n"); return c3n; } @@ -1253,24 +1278,35 @@ u3e_backup(c3_c* pax_c, c3_o ovw_o) return c3n; } + // open source image files + // + if ( c3n == _ce_image_open(&nux_u, pux_c) ) { + fprintf(stderr, "loom: couldn't open north image at %s\r\n", pux_c); + return c3n; + } + if ( c3n == _ce_image_open(&sux_u, pux_c) ) { + fprintf(stderr, "loom: couldn't open south image at %s\r\n", pux_c); + return c3n; + } + c3_c nop_c[8193]; - snprintf(nop_c, 8192, "%s/%s.bin", pax_c, nop_u.nam_c); + snprintf(nop_c, 8192, "%s/%s.bin", pax_c, nax_u.nam_c); - if ( -1 == (nop_u.fid_i = c3_open(nop_c, mod_i, 0666)) ) { + if ( -1 == (nax_u.fid_i = c3_open(nop_c, mod_i, 0666)) ) { fprintf(stderr, "loom: c3_open %s: %s\r\n", nop_c, strerror(errno)); return c3n; } c3_c sop_c[8193]; - snprintf(sop_c, 8192, "%s/%s.bin", pax_c, sop_u.nam_c); + snprintf(sop_c, 8192, "%s/%s.bin", pax_c, sax_u.nam_c); - if ( -1 == (sop_u.fid_i = c3_open(sop_c, mod_i, 0666)) ) { + if ( -1 == (sax_u.fid_i = c3_open(sop_c, mod_i, 0666)) ) { fprintf(stderr, "loom: c3_open %s: %s\r\n", sop_c, strerror(errno)); return c3n; } - if ( (c3n == _ce_image_copy(&u3P.nor_u, &nop_u)) - || (c3n == _ce_image_copy(&u3P.sou_u, &sop_u)) ) + if ( (c3n == _ce_image_copy(&nux_u, &nax_u)) + || (c3n == _ce_image_copy(&sux_u, &sax_u)) ) { c3_unlink(nop_c); c3_unlink(sop_c); @@ -1278,8 +1314,8 @@ u3e_backup(c3_c* pax_c, c3_o ovw_o) return c3n; } - close(nop_u.fid_i); - close(sop_u.fid_i); + close(nax_u.fid_i); + close(sax_u.fid_i); fprintf(stderr, "loom: image backup complete\r\n"); return c3y; } @@ -1431,8 +1467,10 @@ u3e_live(c3_o nuu_o, c3_c* dir_c) { // Open image files. // - if ( (c3n == _ce_image_open(&u3P.nor_u)) || - (c3n == _ce_image_open(&u3P.sou_u)) ) + c3_c chk_c[8193]; + snprintf(chk_c, 8193, "%s/.urb/chk", u3P.dir_c); + if ( (c3n == _ce_image_open(&u3P.nor_u, chk_c)) || + (c3n == _ce_image_open(&u3P.sou_u, chk_c)) ) { fprintf(stderr, "boot: image failed\r\n"); exit(1); diff --git a/pkg/noun/events.h b/pkg/noun/events.h index d3a2b58a0b..462372cd28 100644 --- a/pkg/noun/events.h +++ b/pkg/noun/events.h @@ -73,10 +73,11 @@ /** Functions. **/ - /* u3e_backup(): copy the snapshot to pax_c, overwrite optional. + /* u3e_backup(): copy the snapshot from [pux_c] to [pax_c], + * overwriting optional. */ c3_o - u3e_backup(c3_c* pax_c, c3_o ovw_o); + u3e_backup(c3_c* pux_c, c3_c* pax_c, c3_o ovw_o); /* u3e_fault(): handle a memory fault. */ diff --git a/pkg/vere/disk.c b/pkg/vere/disk.c index b06ea35fec..103f85a484 100644 --- a/pkg/vere/disk.c +++ b/pkg/vere/disk.c @@ -1152,9 +1152,11 @@ u3_disk_epoc_init(u3_disk* log_u, c3_d epo_d) fprintf(biv_f, URBIT_VERSION); fclose(biv_f); - // copy snapshot (skip if first epoch) + // copy snapshot files (skip if first epoch) if ( epo_d > 0 ) { - if ( c3n == u3e_backup(epo_c, c3y) ) { + c3_c chk_c[8193]; + snprintf(chk_c, 8192, "%s/.urb/chk", u3_Host.dir_c); + if ( c3n == u3e_backup(epo_c, chk_c, c3y) ) { fprintf(stderr, "disk: failed to copy snapshot to new epoch\r\n"); goto fail; } diff --git a/pkg/vere/main.c b/pkg/vere/main.c index 0f552b0252..e3da6e9291 100644 --- a/pkg/vere/main.c +++ b/pkg/vere/main.c @@ -2118,20 +2118,19 @@ _cw_play(c3_i argc, c3_c* argv[]) // his need for a serious surgical operation // - // XX this should restore the epoch snapshot and replay that - // if ( c3y == ful_o ) { + // copy the latest epoch's snapshot files into chk/ u3l_log("mars: preparing for full replay"); - u3m_init((size_t)1 << u3_Host.ops_u.lom_y); - u3e_live(c3n, u3_Host.dir_c); - u3m_foul(); - u3m_pave(c3y); - u3j_boot(c3y); - } - else { - u3m_boot(u3_Host.dir_c, (size_t)1 << u3_Host.ops_u.lom_y); + c3_c chk_c[8193]; + snprintf(chk_c, 8193, "%s/.urb/chk", u3_Host.dir_c); + if ( 0 != u3e_backup(log_u->epo_u->pax_c, chk_c, c3y) ) { + fprintf(stderr, "mars: failed to copy snapshot\r\n"); + exit(1); + } } + u3m_boot(u3_Host.dir_c, (size_t)1 << u3_Host.ops_u.lom_y); + u3C.slog_f = _cw_play_slog; { From c9a624d2b299d290ee97cd1699f9b9e3d2e9032d Mon Sep 17 00:00:00 2001 From: Matthew LeVan Date: Wed, 21 Jun 2023 12:59:03 -0400 Subject: [PATCH 052/271] events: better path naming --- pkg/noun/events.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/pkg/noun/events.c b/pkg/noun/events.c index 6da639e476..f9c3904432 100644 --- a/pkg/noun/events.c +++ b/pkg/noun/events.c @@ -1289,27 +1289,27 @@ u3e_backup(c3_c* pux_c, c3_c* pax_c, c3_o ovw_o) return c3n; } - c3_c nop_c[8193]; - snprintf(nop_c, 8192, "%s/%s.bin", pax_c, nax_u.nam_c); + c3_c nax_c[8193]; + snprintf(nax_c, 8192, "%s/%s.bin", pax_c, nax_u.nam_c); - if ( -1 == (nax_u.fid_i = c3_open(nop_c, mod_i, 0666)) ) { - fprintf(stderr, "loom: c3_open %s: %s\r\n", nop_c, strerror(errno)); + if ( -1 == (nax_u.fid_i = c3_open(nax_c, mod_i, 0666)) ) { + fprintf(stderr, "loom: c3_open %s: %s\r\n", nax_c, strerror(errno)); return c3n; } - c3_c sop_c[8193]; - snprintf(sop_c, 8192, "%s/%s.bin", pax_c, sax_u.nam_c); + c3_c sax_c[8193]; + snprintf(sax_c, 8192, "%s/%s.bin", pax_c, sax_u.nam_c); - if ( -1 == (sax_u.fid_i = c3_open(sop_c, mod_i, 0666)) ) { - fprintf(stderr, "loom: c3_open %s: %s\r\n", sop_c, strerror(errno)); + if ( -1 == (sax_u.fid_i = c3_open(sax_c, mod_i, 0666)) ) { + fprintf(stderr, "loom: c3_open %s: %s\r\n", sax_c, strerror(errno)); return c3n; } if ( (c3n == _ce_image_copy(&nux_u, &nax_u)) || (c3n == _ce_image_copy(&sux_u, &sax_u)) ) { - c3_unlink(nop_c); - c3_unlink(sop_c); + c3_unlink(nax_c); + c3_unlink(sax_c); fprintf(stderr, "loom: image backup failed\r\n"); return c3n; } From 01ecd2e40fa4d828d4614c092b61825d442d5b41 Mon Sep 17 00:00:00 2001 From: Matthew LeVan Date: Thu, 22 Jun 2023 09:41:51 -0400 Subject: [PATCH 053/271] disk: fix merge error --- pkg/vere/disk.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/vere/disk.c b/pkg/vere/disk.c index 73f92ac43a..7bfb89fff8 100644 --- a/pkg/vere/disk.c +++ b/pkg/vere/disk.c @@ -1156,7 +1156,7 @@ u3_disk_epoc_init(u3_disk* log_u, c3_d epo_d) if ( epo_d > 0 ) { c3_c chk_c[8193]; snprintf(chk_c, 8192, "%s/.urb/chk", u3_Host.dir_c); - if ( c3n == u3e_backup(epo_c, chk_c, c3y) ) { + if ( c3n == u3e_backup(chk_c, epo_c, c3y) ) { fprintf(stderr, "disk: failed to copy snapshot to new epoch\r\n"); goto fail; } From 4336422ae061abd7cc234d2cb6abbdfc0d14e75e Mon Sep 17 00:00:00 2001 From: Matthew LeVan Date: Thu, 22 Jun 2023 14:38:23 -0400 Subject: [PATCH 054/271] events: remove unnecessary/erroneous `mkdir -p` implementation --- pkg/noun/events.c | 27 --------------------------- 1 file changed, 27 deletions(-) diff --git a/pkg/noun/events.c b/pkg/noun/events.c index f3fd13e5ab..afc1d04001 100644 --- a/pkg/noun/events.c +++ b/pkg/noun/events.c @@ -402,33 +402,6 @@ _ce_image_open(u3e_image* img_u, c3_c* ful_c) { c3_i mod_i = O_RDWR | O_CREAT; - // create directories if necessary - // - { - c3_c pax_c[8192]; - c3_c* end_c = pax_c + 8192; - c3_c* cur_c = pax_c + snprintf(pax_c, 8192, "%s", ful_c); - - while ( cur_c < end_c ) { - c3_c* nxt_c = strchr(cur_c, '/'); - - if ( !nxt_c ) { - break; - } - else { - *nxt_c = 0; - - if ( 0 != mkdir(pax_c, 0777) && EEXIST != errno ) { - fprintf(stderr, "loom: mkdir %s: %s\r\n", pax_c, strerror(errno)); - return c3n; - } - - *nxt_c = '/'; - cur_c = nxt_c + 1; - } - } - } - c3_c pax_c[8192]; snprintf(pax_c, 8192, "%s/%s.bin", ful_c, img_u->nam_c); if ( -1 == (img_u->fid_i = c3_open(pax_c, mod_i, 0666)) ) { From ac3dd768a7fce2e057ab13ccf965d6dd248bc183 Mon Sep 17 00:00:00 2001 From: Matthew LeVan Date: Thu, 22 Jun 2023 14:41:44 -0400 Subject: [PATCH 055/271] trim trailing whitespace --- pkg/noun/events.c | 4 ++-- pkg/noun/events.h | 4 ++-- pkg/vere/main.c | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/pkg/noun/events.c b/pkg/noun/events.c index afc1d04001..3078345e16 100644 --- a/pkg/noun/events.c +++ b/pkg/noun/events.c @@ -1336,8 +1336,8 @@ _ce_image_copy(u3e_image* fom_u, u3e_image* tou_u) return c3y; } -/* u3e_backup(): copy snapshot from [pux_c] to [pax_c], - * overwriting optionally. note that image files must +/* u3e_backup(): copy snapshot from [pux_c] to [pax_c], + * overwriting optionally. note that image files must * be named "north" and "south". */ c3_o diff --git a/pkg/noun/events.h b/pkg/noun/events.h index 0d6a1e351b..7442c9df94 100644 --- a/pkg/noun/events.h +++ b/pkg/noun/events.h @@ -74,10 +74,10 @@ /** Functions. **/ - /* u3e_backup(): copy the snapshot from [pux_c] to [pax_c], + /* u3e_backup(): copy the snapshot from [pux_c] to [pax_c], * overwriting optional. */ - c3_o + c3_o u3e_backup(c3_c* pux_c, c3_c* pax_c, c3_o ovw_o); /* u3e_fault(): handle a memory fault. diff --git a/pkg/vere/main.c b/pkg/vere/main.c index 5df8125eef..cb2b59f02c 100644 --- a/pkg/vere/main.c +++ b/pkg/vere/main.c @@ -2234,11 +2234,11 @@ _cw_play(c3_i argc, c3_c* argv[]) // replaying events with the epoch system // - // there should be one process that accounts for what is actually available + // there should be one process that accounts for what is actually available // in the pier and replays the least events possible to produce a valid, // up-to-date snapshot. // - // this replay process also uses the chk/ folder as a place to + // this replay process also uses the chk/ folder as a place to // "accumulate" snapshots over the course of a replay if it needs to, // overwriting the existing .bin files as it goes. // @@ -2250,7 +2250,7 @@ _cw_play(c3_i argc, c3_c* argv[]) // i. the snapshot in the latest epoch // 1. if it is valid, replay the events and save the resulting // snapshot into chk/, exit 0 or boot into live mode as before - // 2. if it is invalid, return an error that informs the user of + // 2. if it is invalid, return an error that informs the user of // his need for a serious surgical operation // From 6c6ac0c0c4c4c37c0cc01790cdc41b402e79c5c2 Mon Sep 17 00:00:00 2001 From: Matthew LeVan Date: Thu, 22 Jun 2023 20:31:40 -0400 Subject: [PATCH 056/271] events: `mkdir -p` for `_ce_image_open` (needed for boot) --- pkg/noun/events.c | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/pkg/noun/events.c b/pkg/noun/events.c index 3078345e16..8d7910eb26 100644 --- a/pkg/noun/events.c +++ b/pkg/noun/events.c @@ -402,6 +402,23 @@ _ce_image_open(u3e_image* img_u, c3_c* ful_c) { c3_i mod_i = O_RDWR | O_CREAT; + // create directories recursively like `mkdir -p` + // + { + c3_c* dir_c = ful_c; + c3_c* end_c = ful_c + strlen(ful_c); + + while ( dir_c < end_c ) { + if ( '/' == *dir_c ) { + *dir_c = 0; + c3_mkdir(ful_c, 0700); + *dir_c = '/'; + } + dir_c++; + } + c3_mkdir(ful_c, 0700); + } + c3_c pax_c[8192]; snprintf(pax_c, 8192, "%s/%s.bin", ful_c, img_u->nam_c); if ( -1 == (img_u->fid_i = c3_open(pax_c, mod_i, 0666)) ) { From 64c54543315263698d50a333b8084add9b830b1f Mon Sep 17 00:00:00 2001 From: Matthew LeVan Date: Thu, 22 Jun 2023 20:45:42 -0400 Subject: [PATCH 057/271] events: prevent `0B` snapshot files from being created during `play` --- pkg/noun/events.c | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/pkg/noun/events.c b/pkg/noun/events.c index 8d7910eb26..8dcf529c09 100644 --- a/pkg/noun/events.c +++ b/pkg/noun/events.c @@ -1382,13 +1382,21 @@ u3e_backup(c3_c* pux_c, c3_c* pax_c, c3_o ovw_o) return c3n; } - // open source image files + // open source image files if they exist // - if ( c3n == _ce_image_open(&nux_u, pux_c) ) { + + // check if file exists with access + // + + c3_c nux_c[8193]; + snprintf(nux_c, 8192, "%s/%s.bin", pux_c, nux_u.nam_c); + if ( (0 < access(nux_c, F_OK)) && (c3n == _ce_image_open(&nux_u, pux_c)) ) { fprintf(stderr, "loom: couldn't open north image at %s\r\n", pux_c); return c3n; } - if ( c3n == _ce_image_open(&sux_u, pux_c) ) { + c3_c sux_c[8193]; + snprintf(sux_c, 8192, "%s/%s.bin", pux_c, sux_u.nam_c); + if ( (0 < access(sux_c, F_OK)) && (c3n == _ce_image_open(&sux_u, pux_c)) ) { fprintf(stderr, "loom: couldn't open south image at %s\r\n", pux_c); return c3n; } From b6541b73dc6fdc7772bece5d24bf7bccca760837 Mon Sep 17 00:00:00 2001 From: Matthew LeVan Date: Fri, 23 Jun 2023 10:14:09 -0400 Subject: [PATCH 058/271] events: remove `mkdir -p` functionality from `_ce_image_open` --- pkg/noun/events.c | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/pkg/noun/events.c b/pkg/noun/events.c index 8dcf529c09..8e53f30daa 100644 --- a/pkg/noun/events.c +++ b/pkg/noun/events.c @@ -402,23 +402,6 @@ _ce_image_open(u3e_image* img_u, c3_c* ful_c) { c3_i mod_i = O_RDWR | O_CREAT; - // create directories recursively like `mkdir -p` - // - { - c3_c* dir_c = ful_c; - c3_c* end_c = ful_c + strlen(ful_c); - - while ( dir_c < end_c ) { - if ( '/' == *dir_c ) { - *dir_c = 0; - c3_mkdir(ful_c, 0700); - *dir_c = '/'; - } - dir_c++; - } - c3_mkdir(ful_c, 0700); - } - c3_c pax_c[8192]; snprintf(pax_c, 8192, "%s/%s.bin", ful_c, img_u->nam_c); if ( -1 == (img_u->fid_i = c3_open(pax_c, mod_i, 0666)) ) { @@ -1384,10 +1367,6 @@ u3e_backup(c3_c* pux_c, c3_c* pax_c, c3_o ovw_o) // open source image files if they exist // - - // check if file exists with access - // - c3_c nux_c[8193]; snprintf(nux_c, 8192, "%s/%s.bin", pux_c, nux_u.nam_c); if ( (0 < access(nux_c, F_OK)) && (c3n == _ce_image_open(&nux_u, pux_c)) ) { From 299c6edb02201dbda3c366c86ceee1ca86d7d4f3 Mon Sep 17 00:00:00 2001 From: Matthew LeVan Date: Fri, 23 Jun 2023 10:14:40 -0400 Subject: [PATCH 059/271] disk: create `.urb/chk` on init; simplify `log_u->dun_d` setting --- pkg/vere/disk.c | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/pkg/vere/disk.c b/pkg/vere/disk.c index 7bfb89fff8..b9b5b9d1b7 100644 --- a/pkg/vere/disk.c +++ b/pkg/vere/disk.c @@ -1028,6 +1028,10 @@ u3_disk_init(c3_c* pax_c, u3_disk_cb cb_u) strcat(dir_c, "/.urb/get"); c3_mkdir(dir_c, 0700); + strcpy(dir_c, pax_c); + strcat(dir_c, "/.urb/chk"); + c3_mkdir(dir_c, 0700); + c3_free(dir_c); } @@ -1078,14 +1082,10 @@ u3_disk_init(c3_c* pax_c, u3_disk_cb cb_u) } // initialize dun_d/sen_d values - if ( 0 == las_d ) { // fresh epoch (no events in lmdb yet) - if ( 0 == lat_d ) { // first epoch - log_u->dun_d = 0; - } else { // not first epoch - log_u->dun_d = lat_d; // set dun_d to last event in prev epoch - } - } else { // not fresh epoch - log_u->dun_d = las_d; // set dun_d to last event in lmdb + if ( 0 == las_d ) { // first epoch (no events in lmdb yet) + log_u->dun_d = lat_d; // set dun_d to last event in prev epoch + } else { + log_u->dun_d = las_d; // set dun_d to last event in lmdb } log_u->sen_d = log_u->dun_d; @@ -1307,12 +1307,12 @@ u3_disk_epoc_vere(u3_disk* log_u, c3_d epo_d, c3_c* ver_w) */ c3_o u3_disk_migrate(u3_disk* log_u) { - /* migration steps (* indicates breakpoint set): + /* migration steps: * 0. detect whether we need to migrate or not - * a. if it's a fresh boot via u3_Host.ops_u.nuu -> skip migration (returns yes) - * b. if data.mdb is readable in log directory -> execute migration (returns yes or no) + * a. if it's a fresh boot via u3_Host.ops_u.nuu -> skip migration + * b. if data.mdb is readable in log directory -> execute migration * if not -> skip migration (returns yes) - * 1. initialize epoch 0i0 (first call to u3_disk_epoc_init()) * + * 1. initialize epoch 0i0 (first call to u3_disk_epoc_init()) * a. creates epoch directory * b. creates epoch version file * c. creates binary version file @@ -1321,10 +1321,10 @@ c3_o u3_disk_migrate(u3_disk* log_u) * f. writes metadata to new database * g. loads new epoch directory and sets it in log_u * 2. create hard links to data.mdb and lock.mdb in 0i0/ - * 3. rollover to new epoch (second call to u3_disk_epoc_init()) * + * 3. rollover to new epoch (second call to u3_disk_epoc_init()) * a. same as 1a-g but also copies current snapshot between c/d steps - * 4. delete backup snapshot (c3_unlink() and c3_rmdir() calls) * - * 5. delete old data.mdb and lock.mdb files (c3_unlink() calls) * + * 4. delete backup snapshot (c3_unlink() and c3_rmdir() calls) + * 5. delete old data.mdb and lock.mdb files (c3_unlink() calls) */ // check if data.mdb is readable in log directory From 8a3df1dd3df59b190ebe5e70cffe40fbd278c686 Mon Sep 17 00:00:00 2001 From: Matthew LeVan Date: Tue, 27 Jun 2023 11:06:56 -0400 Subject: [PATCH 060/271] play: full replay when only epoch 0 exists --- pkg/vere/disk.c | 17 ++++++----------- pkg/vere/main.c | 47 ++++++++++++++++++++++------------------------- pkg/vere/vere.h | 2 +- 3 files changed, 29 insertions(+), 37 deletions(-) diff --git a/pkg/vere/disk.c b/pkg/vere/disk.c index b9b5b9d1b7..615634eb30 100644 --- a/pkg/vere/disk.c +++ b/pkg/vere/disk.c @@ -1082,15 +1082,15 @@ u3_disk_init(c3_c* pax_c, u3_disk_cb cb_u) } // initialize dun_d/sen_d values - if ( 0 == las_d ) { // first epoch (no events in lmdb yet) - log_u->dun_d = lat_d; // set dun_d to last event in prev epoch - } else { - log_u->dun_d = las_d; // set dun_d to last event in lmdb - } + log_u->dun_d = ( 0 != las_d ) ? las_d : lat_d; log_u->sen_d = log_u->dun_d; + // mark the latest epoch directory + log_u->epo_d = lat_d; + // if binary version of latest epoch is not the same as the // running binary, then we need to create a new epoch + // XX move this into its own function and call it in `u3_pier_stay()` c3_c ver_c[8193]; if ( c3n == u3_disk_epoc_vere(log_u, lat_d, ver_c) ) { fprintf(stderr, "disk: failed to load epoch version\r\n"); @@ -1105,9 +1105,6 @@ u3_disk_init(c3_c* pax_c, u3_disk_cb cb_u) } } - // mark the latest epoch directory - log_u->epo_u = u3_foil_folder(epo_c); - // mark the log as live log_u->liv_o = c3y; } @@ -1193,7 +1190,7 @@ u3_disk_epoc_init(u3_disk* log_u, c3_d epo_d) } // load new epoch directory and set it in log_u - log_u->epo_u = u3_foil_folder(epo_c); + log_u->epo_d = epo_d; // success return c3y; @@ -1346,8 +1343,6 @@ c3_o u3_disk_migrate(u3_disk* log_u) // if fresh boot, initialize disk v1 if ( c3y == u3_Host.ops_u.nuu ) { - fprintf(stderr, "disk: initializing disk with v%d format\r\n", U3D_VER1); - // initialize first epoch "0i0" if ( c3n == u3_disk_epoc_init(log_u, 0) ) { fprintf(stderr, "disk: failed to initialize first epoch\r\n"); diff --git a/pkg/vere/main.c b/pkg/vere/main.c index cb2b59f02c..ee09777c29 100644 --- a/pkg/vere/main.c +++ b/pkg/vere/main.c @@ -2232,34 +2232,31 @@ _cw_play(c3_i argc, c3_c* argv[]) u3C.wag_w |= u3o_hashless; - // replaying events with the epoch system - // - // there should be one process that accounts for what is actually available - // in the pier and replays the least events possible to produce a valid, - // up-to-date snapshot. - // - // this replay process also uses the chk/ folder as a place to - // "accumulate" snapshots over the course of a replay if it needs to, - // overwriting the existing .bin files as it goes. - // - // 0. try loading the snapshot in chk/ - // a. if it is valid and up-to-date with the log - // i. if we executed via `urbit play`, exit 0 - // ii. if we executed via `urbit`, boot into live mode - // b. if it is invalid, try loading a snapshot from - // i. the snapshot in the latest epoch - // 1. if it is valid, replay the events and save the resulting - // snapshot into chk/, exit 0 or boot into live mode as before - // 2. if it is invalid, return an error that informs the user of - // his need for a serious surgical operation - // - if ( c3y == ful_o ) { - // copy the latest epoch's snapshot files into chk/ u3l_log("mars: preparing for full replay"); - c3_c chk_c[8193]; + + c3_c chk_c[8193], epo_c[8193]; snprintf(chk_c, 8193, "%s/.urb/chk", u3_Host.dir_c); - if ( 0 != u3e_backup(log_u->epo_u->pax_c, chk_c, c3y) ) { + snprintf(epo_c, 8192, "%s/0i%" PRIc3_d, log_u->com_u->pax_c, log_u->epo_d); + + // if epoch 0 is the latest, delete the snapshot files in chk/ + if ( 0 == log_u->epo_d ) { + c3_c nor_c[8193], sop_c[8193]; + snprintf(nor_c, 8193, "%s/.urb/chk/north.bin", u3_Host.dir_c); + snprintf(sop_c, 8193, "%s/.urb/chk/south.bin", u3_Host.dir_c); + if ( c3_unlink(nor_c) && (ENOENT != errno) ) { + fprintf(stderr, "mars: failed to unlink %s: %s\r\n", + nor_c, strerror(errno)); + exit(1); + } + if ( c3_unlink(sop_c) && (ENOENT != errno) ) { + fprintf(stderr, "mars: failed to unlink %s: %s\r\n", + sop_c, strerror(errno)); + exit(1); + } + } + else if ( 0 != u3e_backup(epo_c, chk_c, c3y) ) { + // copy the latest epoch's snapshot files into chk/ fprintf(stderr, "mars: failed to copy snapshot\r\n"); exit(1); } diff --git a/pkg/vere/vere.h b/pkg/vere/vere.h index 4e3ea2757c..58685ac4e6 100644 --- a/pkg/vere/vere.h +++ b/pkg/vere/vere.h @@ -539,11 +539,11 @@ u3_dire* dir_u; // main pier directory u3_dire* urb_u; // urbit system data u3_dire* com_u; // log directory - u3_dire* epo_u; // current epoch directory c3_o liv_o; // live void* mdb_u; // lmdb env of current epoch c3_d sen_d; // commit requested c3_d dun_d; // committed + c3_d epo_d; // current epoch number u3_disk_cb cb_u; // callbacks u3_read* red_u; // read requests union { // write thread/request From fa9cdddda45008feb63fcda1bd3d397ba2deb321 Mon Sep 17 00:00:00 2001 From: Matthew LeVan Date: Wed, 28 Jun 2023 11:23:06 -0400 Subject: [PATCH 061/271] events/disk/pier: implement `u3_disk_vere_diff` --- pkg/noun/events.c | 12 ++++------ pkg/vere/disk.c | 49 ++++++++++++++++++++++++--------------- pkg/vere/main.c | 59 ++++++++++++++++++++++++++--------------------- pkg/vere/pier.c | 10 +++++++- pkg/vere/vere.h | 5 ++++ 5 files changed, 82 insertions(+), 53 deletions(-) diff --git a/pkg/noun/events.c b/pkg/noun/events.c index 8e53f30daa..15db40605a 100644 --- a/pkg/noun/events.c +++ b/pkg/noun/events.c @@ -1344,8 +1344,8 @@ c3_o u3e_backup(c3_c* pux_c, c3_c* pax_c, c3_o ovw_o) { // source image files from [pux_c] - u3e_image nux_u = { .nam_c = "north", .pgs_w = 0 }; - u3e_image sux_u = { .nam_c = "south", .pgs_w = 0 }; + u3e_image nux_u = { .nam_c = "north" }; + u3e_image sux_u = { .nam_c = "south" }; // destination image files to [pax_c] u3e_image nax_u = { .nam_c = "north" }; @@ -1369,28 +1369,26 @@ u3e_backup(c3_c* pux_c, c3_c* pax_c, c3_o ovw_o) // c3_c nux_c[8193]; snprintf(nux_c, 8192, "%s/%s.bin", pux_c, nux_u.nam_c); - if ( (0 < access(nux_c, F_OK)) && (c3n == _ce_image_open(&nux_u, pux_c)) ) { + if ( (0 != access(nux_c, F_OK)) || (c3n == _ce_image_open(&nux_u, pux_c)) ) { fprintf(stderr, "loom: couldn't open north image at %s\r\n", pux_c); return c3n; } c3_c sux_c[8193]; snprintf(sux_c, 8192, "%s/%s.bin", pux_c, sux_u.nam_c); - if ( (0 < access(sux_c, F_OK)) && (c3n == _ce_image_open(&sux_u, pux_c)) ) { + if ( (0 != access(sux_c, F_OK)) || (c3n == _ce_image_open(&sux_u, pux_c)) ) { fprintf(stderr, "loom: couldn't open south image at %s\r\n", pux_c); return c3n; } + // open destination image files c3_c nax_c[8193]; snprintf(nax_c, 8192, "%s/%s.bin", pax_c, nax_u.nam_c); - if ( -1 == (nax_u.fid_i = c3_open(nax_c, mod_i, 0666)) ) { fprintf(stderr, "loom: c3_open %s: %s\r\n", nax_c, strerror(errno)); return c3n; } - c3_c sax_c[8193]; snprintf(sax_c, 8192, "%s/%s.bin", pax_c, sax_u.nam_c); - if ( -1 == (sax_u.fid_i = c3_open(sax_c, mod_i, 0666)) ) { fprintf(stderr, "loom: c3_open %s: %s\r\n", sax_c, strerror(errno)); return c3n; diff --git a/pkg/vere/disk.c b/pkg/vere/disk.c index 615634eb30..4d95a9516e 100644 --- a/pkg/vere/disk.c +++ b/pkg/vere/disk.c @@ -1088,23 +1088,6 @@ u3_disk_init(c3_c* pax_c, u3_disk_cb cb_u) // mark the latest epoch directory log_u->epo_d = lat_d; - // if binary version of latest epoch is not the same as the - // running binary, then we need to create a new epoch - // XX move this into its own function and call it in `u3_pier_stay()` - c3_c ver_c[8193]; - if ( c3n == u3_disk_epoc_vere(log_u, lat_d, ver_c) ) { - fprintf(stderr, "disk: failed to load epoch version\r\n"); - c3_free(log_u); - return 0; - } - if ( 0 != strcmp(ver_c, URBIT_VERSION) ) { - if ( c3n == u3_disk_epoc_init(log_u, log_u->dun_d) ) { - fprintf(stderr, "disk: failed to initialize epoch\r\n"); - c3_free(log_u); - return 0; - } - } - // mark the log as live log_u->liv_o = c3y; } @@ -1416,9 +1399,11 @@ c3_o u3_disk_migrate(u3_disk* log_u) snprintf(sop_c, sizeof(sop_c), "%s/south.bin", bhk_c); if ( c3n == c3_unlink(nop_c) ) { fprintf(stderr, "disk: failed to delete bhk/north.bin\r\n"); - } else if ( c3n == c3_unlink(sop_c) ) { + } + else if ( c3n == c3_unlink(sop_c) ) { fprintf(stderr, "disk: failed to delete bhk/south.bin\r\n"); - } else { + } + else { if ( c3n == c3_rmdir(bhk_c) ) { fprintf(stderr, "disk: failed to delete bhk/\r\n"); } @@ -1438,3 +1423,29 @@ c3_o u3_disk_migrate(u3_disk* log_u) return c3y; } + + +/* u3_disk_vere_diff(): checks if vere version mismatches latest epoch's. +*/ +c3_o +u3_disk_vere_diff(u3_disk* log_u) +{ + c3_d lat_d; + if ( c3n == u3_disk_epoc_last(log_u, &lat_d) ) { + fprintf(stderr, "disk: failed to load last epoch\r\n"); + c3_free(log_u); + return 0; + } + + c3_c ver_c[8193]; + if ( c3n == u3_disk_epoc_vere(log_u, lat_d, ver_c) ) { + fprintf(stderr, "disk: failed to load epoch version\r\n"); + c3_free(log_u); + return 0; + } + if ( 0 != strcmp(ver_c, URBIT_VERSION) ) { + return c3y; + } + + return c3n; +} \ No newline at end of file diff --git a/pkg/vere/main.c b/pkg/vere/main.c index ee09777c29..0cc3b4afc8 100644 --- a/pkg/vere/main.c +++ b/pkg/vere/main.c @@ -2109,6 +2109,38 @@ _cw_play_slog(u3_noun hod) u3z(hod); } +/* _cw_play_snap(): prepare snapshot for full replay. +*/ +static void +_cw_play_snap(u3_disk* log_u) +{ + c3_c chk_c[8193], epo_c[8193]; + snprintf(chk_c, 8193, "%s/.urb/chk", u3_Host.dir_c); + snprintf(epo_c, 8192, "%s/0i%" PRIc3_d, log_u->com_u->pax_c, log_u->epo_d); + + if ( 0 == log_u->epo_d ) { + // if epoch 0 is the latest, delete the snapshot files in chk/ + c3_c nor_c[8193], sop_c[8193]; + snprintf(nor_c, 8193, "%s/.urb/chk/north.bin", u3_Host.dir_c); + snprintf(sop_c, 8193, "%s/.urb/chk/south.bin", u3_Host.dir_c); + if ( c3_unlink(nor_c) && (ENOENT != errno) ) { + fprintf(stderr, "mars: failed to unlink %s: %s\r\n", + nor_c, strerror(errno)); + exit(1); + } + if ( c3_unlink(sop_c) && (ENOENT != errno) ) { + fprintf(stderr, "mars: failed to unlink %s: %s\r\n", + sop_c, strerror(errno)); + exit(1); + } + } + else if ( 0 != u3e_backup(epo_c, chk_c, c3y) ) { + // copy the latest epoch's snapshot files into chk/ + fprintf(stderr, "mars: failed to copy snapshot\r\n"); + exit(1); + } +} + /* _cw_play_exit(): exit immediately. */ static void @@ -2234,32 +2266,7 @@ _cw_play(c3_i argc, c3_c* argv[]) if ( c3y == ful_o ) { u3l_log("mars: preparing for full replay"); - - c3_c chk_c[8193], epo_c[8193]; - snprintf(chk_c, 8193, "%s/.urb/chk", u3_Host.dir_c); - snprintf(epo_c, 8192, "%s/0i%" PRIc3_d, log_u->com_u->pax_c, log_u->epo_d); - - // if epoch 0 is the latest, delete the snapshot files in chk/ - if ( 0 == log_u->epo_d ) { - c3_c nor_c[8193], sop_c[8193]; - snprintf(nor_c, 8193, "%s/.urb/chk/north.bin", u3_Host.dir_c); - snprintf(sop_c, 8193, "%s/.urb/chk/south.bin", u3_Host.dir_c); - if ( c3_unlink(nor_c) && (ENOENT != errno) ) { - fprintf(stderr, "mars: failed to unlink %s: %s\r\n", - nor_c, strerror(errno)); - exit(1); - } - if ( c3_unlink(sop_c) && (ENOENT != errno) ) { - fprintf(stderr, "mars: failed to unlink %s: %s\r\n", - sop_c, strerror(errno)); - exit(1); - } - } - else if ( 0 != u3e_backup(epo_c, chk_c, c3y) ) { - // copy the latest epoch's snapshot files into chk/ - fprintf(stderr, "mars: failed to copy snapshot\r\n"); - exit(1); - } + _cw_play_snap(log_u); } u3m_boot(u3_Host.dir_c, (size_t)1 << u3_Host.ops_u.lom_y); diff --git a/pkg/vere/pier.c b/pkg/vere/pier.c index 263af55e1a..c2e7c47026 100644 --- a/pkg/vere/pier.c +++ b/pkg/vere/pier.c @@ -782,7 +782,7 @@ _pier_on_lord_wyrd_bail(void* ptr_v, u3_ovum* egg_u, u3_noun lud) #endif } -/* _pier_wyrd_init(): construct %wyrd. +/* _pier_wyrd_card(): construct %wyrd. */ static u3_noun _pier_wyrd_card(u3_pier* pir_u) @@ -825,6 +825,14 @@ _pier_wyrd_card(u3_pier* pir_u) static void _pier_wyrd_init(u3_pier* pir_u) { + // create a new epoch if current version mismatches the latest epoch's + if ( c3y == u3_disk_vere_diff(pir_u->log_u) ) { + if ( c3n == u3_disk_epoc_init(pir_u->log_u, pir_u->log_u->dun_d) ) { + fprintf(stderr, "disk: failed to initialize epoch\r\n"); + exit(1); + } + } + u3_noun cad = _pier_wyrd_card(pir_u); u3_noun wir = u3nc(c3__arvo, u3_nul); diff --git a/pkg/vere/vere.h b/pkg/vere/vere.h index 58685ac4e6..c72417f86d 100644 --- a/pkg/vere/vere.h +++ b/pkg/vere/vere.h @@ -1025,6 +1025,11 @@ c3_o u3_disk_epoc_vere(u3_disk* log_u, c3_d epo_d, c3_c* ver_w); + /* u3_disk_vere_diff(): checks if vere version mismatches latest epoch's. + */ + c3_o + u3_disk_vere_diff(u3_disk* log_u); + /* u3_disk_migrate(): migrates disk format. */ c3_o From 4d35ba25b3300b82a00b3b82b4d76d1cc9581407 Mon Sep 17 00:00:00 2001 From: Matthew LeVan Date: Wed, 28 Jun 2023 11:43:22 -0400 Subject: [PATCH 062/271] pier: move `.urb/chk` creation into `u3m_pier` --- pkg/noun/manage.c | 38 +++++++++++++++++++++++++++++++++++++- pkg/noun/manage.h | 5 +++++ pkg/vere/disk.c | 4 ---- 3 files changed, 42 insertions(+), 5 deletions(-) diff --git a/pkg/noun/manage.c b/pkg/noun/manage.c index 9b2450af12..2e3ce7accd 100644 --- a/pkg/noun/manage.c +++ b/pkg/noun/manage.c @@ -2068,6 +2068,42 @@ u3m_stop() u3je_secp_stop(); } +/* u3m_pier(): make a pier. +*/ +c3_c* +u3m_pier(c3_c* dir_c) +{ + c3_c ful_c[8193]; + + u3C.dir_c = dir_c; + + snprintf(ful_c, 8192, "%s", dir_c); + if ( c3_mkdir(ful_c, 0700) ) { + if ( EEXIST != errno ) { + fprintf(stderr, "loom: pier create: %s\r\n", strerror(errno)); + exit(1); + } + } + + snprintf(ful_c, 8192, "%s/.urb", dir_c); + if ( c3_mkdir(ful_c, 0700) ) { + if ( EEXIST != errno ) { + fprintf(stderr, "loom: .urb create: %s\r\n", strerror(errno)); + exit(1); + } + } + + snprintf(ful_c, 8192, "%s/.urb/chk", dir_c); + if ( c3_mkdir(ful_c, 0700) ) { + if ( EEXIST != errno ) { + fprintf(stderr, "loom: .urb/chk create: %s\r\n", strerror(errno)); + exit(1); + } + } + + return strdup(dir_c); +} + /* u3m_boot(): start the u3 system. return next event, starting from 1. */ c3_d @@ -2083,7 +2119,7 @@ u3m_boot(c3_c* dir_c, size_t len_i) /* Activate the storage system. */ - nuu_o = u3e_live(c3n, dir_c); + nuu_o = u3e_live(c3n, u3m_pier(dir_c)); /* Activate tracing. */ diff --git a/pkg/noun/manage.h b/pkg/noun/manage.h index 297585c486..56d4ca7901 100644 --- a/pkg/noun/manage.h +++ b/pkg/noun/manage.h @@ -14,6 +14,11 @@ c3_d u3m_boot(c3_c* dir_c, size_t len_i); + /* u3m_pier(): make a pier. + */ + c3_c* + u3m_pier(c3_c* dir_c); + /* u3m_boot_lite(): start without checkpointing. */ c3_d diff --git a/pkg/vere/disk.c b/pkg/vere/disk.c index 4d95a9516e..2085262bd7 100644 --- a/pkg/vere/disk.c +++ b/pkg/vere/disk.c @@ -1028,10 +1028,6 @@ u3_disk_init(c3_c* pax_c, u3_disk_cb cb_u) strcat(dir_c, "/.urb/get"); c3_mkdir(dir_c, 0700); - strcpy(dir_c, pax_c); - strcat(dir_c, "/.urb/chk"); - c3_mkdir(dir_c, 0700); - c3_free(dir_c); } From 8668d732bbed6896d800e6bccf12687d47160d1d Mon Sep 17 00:00:00 2001 From: Matthew LeVan Date: Thu, 29 Jun 2023 21:11:38 -0400 Subject: [PATCH 063/271] cli: subcommands auto-migrate disk format --- pkg/vere/main.c | 37 +++++++++++-------------------------- 1 file changed, 11 insertions(+), 26 deletions(-) diff --git a/pkg/vere/main.c b/pkg/vere/main.c index 0cc3b4afc8..2291ed3b4f 100644 --- a/pkg/vere/main.c +++ b/pkg/vere/main.c @@ -1561,11 +1561,11 @@ _cw_info(c3_i argc, c3_c* argv[]) exit(1); } - c3_d eve_d = u3m_boot(u3_Host.dir_c, (size_t)1 << u3_Host.ops_u.lom_y); + u3_Host.eve_d = u3m_boot(u3_Host.dir_c, (size_t)1 << u3_Host.ops_u.lom_y); u3_disk* log_u = _cw_disk_init(u3_Host.dir_c); fprintf(stderr, "\r\nurbit: %s at event %" PRIu64 "\r\n", - u3_Host.dir_c, eve_d); + u3_Host.dir_c, u3_Host.eve_d); u3_disk_slog(log_u); printf("\n"); @@ -1720,17 +1720,17 @@ _cw_cram(c3_i argc, c3_c* argv[]) exit(1); } - c3_d eve_d = u3m_boot(u3_Host.dir_c, (size_t)1 << u3_Host.ops_u.lom_y); + u3_Host.eve_d = u3m_boot(u3_Host.dir_c, (size_t)1 << u3_Host.ops_u.lom_y); u3_disk* log_u = _cw_disk_init(u3_Host.dir_c); // XX s/b try_aquire lock c3_o ret_o; fprintf(stderr, "urbit: cram: preparing\r\n"); - if ( c3n == (ret_o = u3u_cram(u3_Host.dir_c, eve_d)) ) { + if ( c3n == (ret_o = u3u_cram(u3_Host.dir_c, u3_Host.eve_d)) ) { fprintf(stderr, "urbit: cram: unable to jam state\r\n"); } else { - fprintf(stderr, "urbit: cram: rock saved at event %" PRIu64 "\r\n", eve_d); + fprintf(stderr, "urbit: cram: rock saved at event %" PRIu64 "\r\n", u3_Host.eve_d); } // save even on failure, as we just did all the work of deduplication @@ -1832,12 +1832,11 @@ _cw_queu(c3_i argc, c3_c* argv[]) exit(1); } else { + u3_Host.eve_d = u3m_boot(u3_Host.dir_c, (size_t)1 << u3_Host.ops_u.lom_y); u3_disk* log_u = _cw_disk_init(u3_Host.dir_c); // XX s/b try_aquire lock fprintf(stderr, "urbit: queu: preparing\r\n"); - u3m_boot(u3_Host.dir_c, (size_t)1 << u3_Host.ops_u.lom_y); - // XX can spuriously fail do to corrupt memory-image checkpoint, // need a u3m_half_boot equivalent // workaround is to delete/move the checkpoint in case of corruption @@ -1925,11 +1924,11 @@ _cw_meld(c3_i argc, c3_c* argv[]) exit(1); } + u3_Host.eve_d = u3m_boot(u3_Host.dir_c, (size_t)1 << u3_Host.ops_u.lom_y); u3_disk* log_u = _cw_disk_init(u3_Host.dir_c); // XX s/b try_aquire lock c3_w pre_w; u3C.wag_w |= u3o_hashless; - u3m_boot(u3_Host.dir_c, (size_t)1 << u3_Host.ops_u.lom_y); pre_w = u3a_open(u3R); u3u_meld(); @@ -2090,9 +2089,9 @@ _cw_pack(c3_i argc, c3_c* argv[]) exit(1); } + u3_Host.eve_d = u3m_boot(u3_Host.dir_c, (size_t)1 << u3_Host.ops_u.lom_y); u3_disk* log_u = _cw_disk_init(u3_Host.dir_c); // XX s/b try_aquire lock - u3m_boot(u3_Host.dir_c, (size_t)1 << u3_Host.ops_u.lom_y); u3a_print_memory(stderr, "urbit: pack: gained", u3m_pack()); u3m_save(); @@ -2246,6 +2245,7 @@ _cw_play(c3_i argc, c3_c* argv[]) // XX handle SIGTSTP so that the lockfile is not orphaned? // + u3_Host.eve_d = u3m_boot(u3_Host.dir_c, (size_t)1 << u3_Host.ops_u.lom_y); u3_disk* log_u = _cw_disk_init(u3_Host.dir_c); // XX s/b try_aquire lock // Handle SIGTSTP as if it was SIGINT. @@ -2269,8 +2269,6 @@ _cw_play(c3_i argc, c3_c* argv[]) _cw_play_snap(log_u); } - u3m_boot(u3_Host.dir_c, (size_t)1 << u3_Host.ops_u.lom_y); - u3C.slog_f = _cw_play_slog; { @@ -2435,19 +2433,9 @@ _cw_chop(c3_i argc, c3_c* argv[]) } // gracefully shutdown the pier if it's running + u3_Host.eve_d = u3m_boot(u3_Host.dir_c, (size_t)1 << u3_Host.ops_u.lom_y); u3_disk* log_u = _cw_disk_init(u3_Host.dir_c); - // note: this includes patch applications (if any) - u3m_boot(u3_Host.dir_c, (size_t)1 << u3_Host.ops_u.lom_y); - - // check if there's a *current* snapshot - if ( log_u->dun_d != u3A->eve_d ) { - fprintf(stderr, "chop: error: snapshot is out of date, please " - "start/shutdown your pier gracefully first\r\n"); - fprintf(stderr, "chop: eve_d: %" PRIc3_d ", dun_d: %" PRIc3_d "\r\n", u3A->eve_d, log_u->dun_d); - exit(1); - } - // get latest epoch number prior to creating a new one c3_d pre_d; if ( c3n == u3_disk_epoc_last(log_u, &pre_d) ) { @@ -2523,7 +2511,6 @@ _cw_chop(c3_i argc, c3_c* argv[]) // cleanup u3_dire_free(ned_u); u3_disk_exit(log_u); - u3m_stop(); // success fprintf(stderr, "chop: event log truncation complete\r\n"); @@ -2580,11 +2567,9 @@ _cw_roll(c3_i argc, c3_c* argv[]) } // gracefully shutdown the pier if it's running + u3_Host.eve_d = u3m_boot(u3_Host.dir_c, (size_t)1 << u3_Host.ops_u.lom_y); u3_disk* log_u = _cw_disk_init(u3_Host.dir_c); - // note: this includes patch applications (if any) - u3m_boot(u3_Host.dir_c, (size_t)1 << u3_Host.ops_u.lom_y); - // check if there's a *current* snapshot if ( log_u->dun_d != u3A->eve_d ) { fprintf(stderr, "roll: error: snapshot is out of date, please " From 5e3f25358bad01beb63f50bf5aaca5d02c3bd403 Mon Sep 17 00:00:00 2001 From: Matthew LeVan Date: Sun, 2 Jul 2023 07:57:06 -0400 Subject: [PATCH 064/271] cli: fix `play` boot --- pkg/vere/main.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/pkg/vere/main.c b/pkg/vere/main.c index 2291ed3b4f..fd7f6703d4 100644 --- a/pkg/vere/main.c +++ b/pkg/vere/main.c @@ -342,7 +342,7 @@ _main_getopt(c3_i argc, c3_c** argv) return c3n; } else { u3_Host.ops_u.sap_w = arg_w * 60; - if ( 0 == u3_Host.ops_u.sap_w) + if ( 0 == u3_Host.ops_u.sap_w ) return c3n; } break; @@ -1197,7 +1197,7 @@ _cw_disk_init(c3_c* dir_c) u3_disk* log_u = u3_disk_init(dir_c, cb_u); if ( !log_u ) { - fprintf(stderr, "unable to open event log\n");; + fprintf(stderr, "unable to open event log\n"); exit(1); } @@ -1924,12 +1924,12 @@ _cw_meld(c3_i argc, c3_c* argv[]) exit(1); } + u3C.wag_w |= u3o_hashless; + u3_Host.eve_d = u3m_boot(u3_Host.dir_c, (size_t)1 << u3_Host.ops_u.lom_y); u3_disk* log_u = _cw_disk_init(u3_Host.dir_c); // XX s/b try_aquire lock c3_w pre_w; - u3C.wag_w |= u3o_hashless; - pre_w = u3a_open(u3R); u3u_meld(); u3a_print_memory(stderr, "urbit: meld: gained", u3u_meld()); @@ -2245,7 +2245,6 @@ _cw_play(c3_i argc, c3_c* argv[]) // XX handle SIGTSTP so that the lockfile is not orphaned? // - u3_Host.eve_d = u3m_boot(u3_Host.dir_c, (size_t)1 << u3_Host.ops_u.lom_y); u3_disk* log_u = _cw_disk_init(u3_Host.dir_c); // XX s/b try_aquire lock // Handle SIGTSTP as if it was SIGINT. @@ -2269,6 +2268,8 @@ _cw_play(c3_i argc, c3_c* argv[]) _cw_play_snap(log_u); } + u3_Host.eve_d = u3m_boot(u3_Host.dir_c, (size_t)1 << u3_Host.ops_u.lom_y); + u3C.slog_f = _cw_play_slog; { From 1140c4e0da4fff6c13ec3a83c1b28342c092d175 Mon Sep 17 00:00:00 2001 From: Matthew LeVan Date: Wed, 5 Jul 2023 14:55:42 -0400 Subject: [PATCH 065/271] play: support full replay of unmigrated piers without valid snapshot --- pkg/vere/disk.c | 20 ++++++++++++++++---- pkg/vere/main.c | 14 +++++++++++--- 2 files changed, 27 insertions(+), 7 deletions(-) diff --git a/pkg/vere/disk.c b/pkg/vere/disk.c index 2085262bd7..56f9a81a30 100644 --- a/pkg/vere/disk.c +++ b/pkg/vere/disk.c @@ -1043,11 +1043,23 @@ u3_disk_init(c3_c* pax_c, u3_disk_cb cb_u) return 0; } - // migrate to the correct disk format if necessary + // try migrating the disk format if ( c3n == u3_disk_migrate(log_u) ) { - fprintf(stderr, "disk: failed to migrate to v%d\r\n", U3D_VER1); - c3_free(log_u); - return 0; + fprintf(stderr, "disk: loading old format\r\n"); + + if ( 0 == (log_u->mdb_u = u3_lmdb_init(log_c, siz_i)) ) { + fprintf(stderr, "disk: failed to initialize lmdb\r\n"); + c3_free(log_u); + } + + c3_d fir_d; + if ( c3n == u3_lmdb_gulf(log_u->mdb_u, &fir_d, &log_u->dun_d) ) { + fprintf(stderr, "disk: failed to load latest event from lmdb\r\n"); + c3_free(log_u); + return 0; + } + + return log_u; } // get latest epoch number diff --git a/pkg/vere/main.c b/pkg/vere/main.c index fd7f6703d4..36cdd6c456 100644 --- a/pkg/vere/main.c +++ b/pkg/vere/main.c @@ -1924,12 +1924,12 @@ _cw_meld(c3_i argc, c3_c* argv[]) exit(1); } - u3C.wag_w |= u3o_hashless; - u3_Host.eve_d = u3m_boot(u3_Host.dir_c, (size_t)1 << u3_Host.ops_u.lom_y); u3_disk* log_u = _cw_disk_init(u3_Host.dir_c); // XX s/b try_aquire lock c3_w pre_w; + u3C.wag_w |= u3o_hashless; + pre_w = u3a_open(u3R); u3u_meld(); u3a_print_memory(stderr, "urbit: meld: gained", u3u_meld()); @@ -2245,7 +2245,11 @@ _cw_play(c3_i argc, c3_c* argv[]) // XX handle SIGTSTP so that the lockfile is not orphaned? // - u3_disk* log_u = _cw_disk_init(u3_Host.dir_c); // XX s/b try_aquire lock + u3_disk* log_u; + if ( 0 == (log_u = _cw_disk_init(u3_Host.dir_c)) ) { + fprintf(stderr, "mars: failed to load event log\r\n"); + exit(1); + } // Handle SIGTSTP as if it was SIGINT. // @@ -2281,6 +2285,10 @@ _cw_play(c3_i argc, c3_c* argv[]) }; u3_mars_play(&mar_u, eve_d, sap_d); + + // migrate after replay, if necessary + u3_Host.eve_d = mar_u.dun_d; + u3_disk_migrate(log_u); } u3_disk_exit(log_u); From fcd9bcc74c59a8d2f0c6762ed4be24dab12e9757 Mon Sep 17 00:00:00 2001 From: Matthew LeVan Date: Thu, 6 Jul 2023 17:09:42 -0400 Subject: [PATCH 066/271] always replay synchronously on restart --- pkg/vere/disk.c | 71 +++++++++++++++++++---------- pkg/vere/main.c | 119 +++++++++++++++++++++++++++--------------------- pkg/vere/pier.c | 2 +- pkg/vere/vere.h | 8 +++- 4 files changed, 120 insertions(+), 80 deletions(-) diff --git a/pkg/vere/disk.c b/pkg/vere/disk.c index 56f9a81a30..cc07da2827 100644 --- a/pkg/vere/disk.c +++ b/pkg/vere/disk.c @@ -975,7 +975,7 @@ u3_disk_slog(u3_disk* log_u) /* u3_disk_init(): load or create pier directories and event log. */ u3_disk* -u3_disk_init(c3_c* pax_c, u3_disk_cb cb_u) +u3_disk_init(c3_c* pax_c, u3_disk_cb cb_u, c3_o mig_o) { u3_disk* log_u = c3_calloc(sizeof(*log_u)); log_u->liv_o = c3n; @@ -1043,23 +1043,33 @@ u3_disk_init(c3_c* pax_c, u3_disk_cb cb_u) return 0; } - // try migrating the disk format - if ( c3n == u3_disk_migrate(log_u) ) { - fprintf(stderr, "disk: loading old format\r\n"); - - if ( 0 == (log_u->mdb_u = u3_lmdb_init(log_c, siz_i)) ) { - fprintf(stderr, "disk: failed to initialize lmdb\r\n"); - c3_free(log_u); + if ( c3y == u3_disk_need_migrate(log_u) ) { + if ( c3y == mig_o ) { + if ( c3n == u3_disk_migrate(log_u) ) { + fprintf(stderr, "disk: failed to migrate log\r\n"); + c3_free(log_u); + return 0; + } } + else { + fprintf(stderr, "disk: loading old format\r\n"); - c3_d fir_d; - if ( c3n == u3_lmdb_gulf(log_u->mdb_u, &fir_d, &log_u->dun_d) ) { - fprintf(stderr, "disk: failed to load latest event from lmdb\r\n"); - c3_free(log_u); - return 0; - } + if ( 0 == (log_u->mdb_u = u3_lmdb_init(log_c, siz_i)) ) { + fprintf(stderr, "disk: failed to initialize lmdb\r\n"); + c3_free(log_u); + } - return log_u; + c3_d fir_d; + if ( c3n == u3_lmdb_gulf(log_u->mdb_u, &fir_d, &log_u->dun_d) ) { + fprintf(stderr, "disk: failed to load latest event from lmdb\r\n"); + c3_free(log_u); + return 0; + } + + log_u->sen_d = log_u->dun_d; + + return log_u; + } } // get latest epoch number @@ -1291,9 +1301,27 @@ u3_disk_epoc_vere(u3_disk* log_u, c3_d epo_d, c3_c* ver_w) return c3y; } +/* u3_disk_need_migrate: does the desk need to be migrated? +*/ +c3_o +u3_disk_need_migrate(u3_disk* log_u) +{ + // check if data.mdb is readable in log directory + c3_c dut_c[8193]; + snprintf(dut_c, sizeof(dut_c), "%s/data.mdb", log_u->com_u->pax_c); + if ( !_(u3_Host.ops_u.nuu) + && 0 != access(dut_c, R_OK) ) { + // if .urb/log/data.mdb is not readable, skip migration + return c3n; + } + + return c3y; +} + /* u3_disk_migrate: migrates disk format. */ -c3_o u3_disk_migrate(u3_disk* log_u) +c3_o +u3_disk_migrate(u3_disk* log_u) { /* migration steps: * 0. detect whether we need to migrate or not @@ -1315,14 +1343,6 @@ c3_o u3_disk_migrate(u3_disk* log_u) * 5. delete old data.mdb and lock.mdb files (c3_unlink() calls) */ - // check if data.mdb is readable in log directory - c3_c dut_c[8193]; - snprintf(dut_c, sizeof(dut_c), "%s/data.mdb", log_u->com_u->pax_c); - if ( !_(u3_Host.ops_u.nuu) - && 0 != access(dut_c, R_OK) ) { - // if .urb/log/data.mdb is not readable, skip migration - return c3y; - } // check if lock.mdb is readable in log directory c3_o luk_o = c3n; @@ -1378,8 +1398,9 @@ c3_o u3_disk_migrate(u3_disk* log_u) } // create hard links to data.mdb and lock.mdb in 0i0/ - c3_c epo_c[8193], dat_c[8193], lok_c[8193]; + c3_c epo_c[8193], dut_c[8193], dat_c[8193], lok_c[8193]; snprintf(epo_c, sizeof(epo_c), "%s/0i0", log_u->com_u->pax_c); + snprintf(dut_c, sizeof(dut_c), "%s/data.mdb", log_u->com_u->pax_c); snprintf(dat_c, sizeof(dat_c), "%s/data.mdb", epo_c); snprintf(lok_c, sizeof(lok_c), "%s/lock.mdb", epo_c); diff --git a/pkg/vere/main.c b/pkg/vere/main.c index 36cdd6c456..abda7a5f14 100644 --- a/pkg/vere/main.c +++ b/pkg/vere/main.c @@ -1194,7 +1194,7 @@ static u3_disk* _cw_disk_init(c3_c* dir_c) { u3_disk_cb cb_u = {0}; - u3_disk* log_u = u3_disk_init(dir_c, cb_u); + u3_disk* log_u = u3_disk_init(dir_c, cb_u, c3y); if ( !log_u ) { fprintf(stderr, "unable to open event log\n"); @@ -2151,6 +2151,69 @@ _cw_play_exit(c3_i int_i) raise(SIGINT); } +/* _cw_play_impl(): replay events, but better. +*/ +static void +_cw_play_impl(c3_d eve_d, c3_d sap_d, c3_o mel_o, c3_o sof_o, c3_o ful_o) +{ + // XX factor this into its own function for calling here and + // from main() (urbit play and boot replay) + + // XX handle SIGTSTP so that the lockfile is not orphaned? + // + u3_disk* log_u; + if ( 0 == (log_u = u3_disk_init(u3_Host.dir_c, (u3_disk_cb){0}, c3n)) ) { + fprintf(stderr, "mars: failed to load event log\r\n"); + exit(1); + } + + // Handle SIGTSTP as if it was SIGINT. + // + // Configured here using signal() so as to be immediately available. + // + signal(SIGTSTP, _cw_play_exit); + + // XX source these from a shared struct ops_u + if ( c3y == mel_o ) { + u3C.wag_w |= u3o_auto_meld; + } + + if ( c3y == sof_o ) { + u3C.wag_w |= u3o_soft_mugs; + } + + u3C.wag_w |= u3o_hashless; + + if ( c3y == ful_o ) { + u3l_log("mars: preparing for full replay"); + _cw_play_snap(log_u); + } + + u3m_boot(u3_Host.dir_c, (size_t)1 << u3_Host.ops_u.lom_y); + + u3C.slog_f = _cw_play_slog; + + { + u3_mars mar_u = { + .log_u = log_u, + .dir_c = u3_Host.dir_c, + .sen_d = u3A->eve_d, + .dun_d = u3A->eve_d, + }; + + u3_mars_play(&mar_u, eve_d, sap_d); + + // migrate after replay, if necessary + u3_Host.eve_d = mar_u.dun_d; + if ( c3y == u3_disk_need_migrate(log_u) ) { + u3_disk_migrate(log_u); + } + } + + u3_disk_exit(log_u); + u3m_stop(); +} + /* _cw_play(): replay events, but better. */ static void @@ -2243,56 +2306,7 @@ _cw_play(c3_i argc, c3_c* argv[]) exit(1); } - // XX handle SIGTSTP so that the lockfile is not orphaned? - // - u3_disk* log_u; - if ( 0 == (log_u = _cw_disk_init(u3_Host.dir_c)) ) { - fprintf(stderr, "mars: failed to load event log\r\n"); - exit(1); - } - - // Handle SIGTSTP as if it was SIGINT. - // - // Configured here using signal() so as to be immediately available. - // - signal(SIGTSTP, _cw_play_exit); - - if ( c3y == mel_o ) { - u3C.wag_w |= u3o_auto_meld; - } - - if ( c3y == sof_o ) { - u3C.wag_w |= u3o_soft_mugs; - } - - u3C.wag_w |= u3o_hashless; - - if ( c3y == ful_o ) { - u3l_log("mars: preparing for full replay"); - _cw_play_snap(log_u); - } - - u3_Host.eve_d = u3m_boot(u3_Host.dir_c, (size_t)1 << u3_Host.ops_u.lom_y); - - u3C.slog_f = _cw_play_slog; - - { - u3_mars mar_u = { - .log_u = log_u, - .dir_c = u3_Host.dir_c, - .sen_d = u3A->eve_d, - .dun_d = u3A->eve_d, - }; - - u3_mars_play(&mar_u, eve_d, sap_d); - - // migrate after replay, if necessary - u3_Host.eve_d = mar_u.dun_d; - u3_disk_migrate(log_u); - } - - u3_disk_exit(log_u); - u3m_stop(); + _cw_play_impl(eve_d, sap_d, mel_o, sof_o, ful_o); } /* _cw_prep(): prepare for upgrade @@ -3085,8 +3099,7 @@ main(c3_i argc, // we need the current snapshot's latest event number to // validate whether we can execute disk migration if ( u3_Host.ops_u.nuu == c3n ) { - u3_Host.eve_d = u3m_boot(u3_Host.dir_c, (size_t)1 << u3_Host.ops_u.lom_y); - u3m_stop(); + _cw_play_impl(0, 0, c3n, c3n, c3n); // XX unmap loom, else parts of the snapshot could be left in memory } diff --git a/pkg/vere/pier.c b/pkg/vere/pier.c index c2e7c47026..81e744e4f7 100644 --- a/pkg/vere/pier.c +++ b/pkg/vere/pier.c @@ -1643,7 +1643,7 @@ _pier_init(c3_w wag_w, c3_c* pax_c) .write_bail_f = _pier_on_disk_write_bail }; - if ( !(pir_u->log_u = u3_disk_init(pax_c, cb_u)) ) { + if ( !(pir_u->log_u = u3_disk_init(pax_c, cb_u, c3n)) ) { c3_free(pir_u); return 0; } diff --git a/pkg/vere/vere.h b/pkg/vere/vere.h index c72417f86d..9cb9e88d8b 100644 --- a/pkg/vere/vere.h +++ b/pkg/vere/vere.h @@ -8,6 +8,7 @@ #include "noun.h" #include "serf.h" #include "uv.h" +#include /** Quasi-tunable parameters. **/ @@ -935,7 +936,7 @@ /* u3_disk_init(): load or create pier directories and event log. */ u3_disk* - u3_disk_init(c3_c* pax_c, u3_disk_cb cb_u); + u3_disk_init(c3_c* pax_c, u3_disk_cb cb_u, c3_o mig_o); /* u3_disk_etch(): serialize an event for persistence. RETAIN [eve] */ @@ -1030,6 +1031,11 @@ c3_o u3_disk_vere_diff(u3_disk* log_u); + /* u3_disk_need_migrate(): does the disk need migration? + */ + c3_o + u3_disk_need_migrate(u3_disk* log_u); + /* u3_disk_migrate(): migrates disk format. */ c3_o From f229c4db9542fc104c3ff527add24a8ac0031fe0 Mon Sep 17 00:00:00 2001 From: Matthew LeVan Date: Thu, 6 Jul 2023 17:13:16 -0400 Subject: [PATCH 067/271] deprecated async replay codepath --- pkg/vere/pier.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/vere/pier.c b/pkg/vere/pier.c index 81e744e4f7..44e9ee4d35 100644 --- a/pkg/vere/pier.c +++ b/pkg/vere/pier.c @@ -1408,6 +1408,8 @@ _pier_on_lord_live(void* ptr_v) // XX print bootstrap commit complete // XX s/b boot_complete_cb // + // XX this codepath should never be hit due to sync replay + u3l_log("pier: warning: async replay"); _pier_play_init(pir_u, log_u->dun_d); } } From 3c8c0b219d3d926a32480149acdf37c8950e9406 Mon Sep 17 00:00:00 2001 From: Matthew LeVan Date: Wed, 12 Jul 2023 12:05:50 -0400 Subject: [PATCH 068/271] disk: fix migrate scenarios --- pkg/vere/disk.c | 4 ++-- pkg/vere/main.c | 3 --- pkg/vere/pier.c | 2 +- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/pkg/vere/disk.c b/pkg/vere/disk.c index cc07da2827..f87ce92154 100644 --- a/pkg/vere/disk.c +++ b/pkg/vere/disk.c @@ -1310,8 +1310,8 @@ u3_disk_need_migrate(u3_disk* log_u) c3_c dut_c[8193]; snprintf(dut_c, sizeof(dut_c), "%s/data.mdb", log_u->com_u->pax_c); if ( !_(u3_Host.ops_u.nuu) - && 0 != access(dut_c, R_OK) ) { - // if .urb/log/data.mdb is not readable, skip migration + && 0 != access(dut_c, F_OK) ) { + // if .urb/log/data.mdb does not exist, skip migration return c3n; } diff --git a/pkg/vere/main.c b/pkg/vere/main.c index 5cc44e0cfa..b09ccdd346 100644 --- a/pkg/vere/main.c +++ b/pkg/vere/main.c @@ -2153,9 +2153,6 @@ _cw_play_exit(c3_i int_i) static void _cw_play_impl(c3_d eve_d, c3_d sap_d, c3_o mel_o, c3_o sof_o, c3_o ful_o) { - // XX factor this into its own function for calling here and - // from main() (urbit play and boot replay) - // XX handle SIGTSTP so that the lockfile is not orphaned? // u3_disk* log_u; diff --git a/pkg/vere/pier.c b/pkg/vere/pier.c index 44e9ee4d35..830a2c1e49 100644 --- a/pkg/vere/pier.c +++ b/pkg/vere/pier.c @@ -1645,7 +1645,7 @@ _pier_init(c3_w wag_w, c3_c* pax_c) .write_bail_f = _pier_on_disk_write_bail }; - if ( !(pir_u->log_u = u3_disk_init(pax_c, cb_u, c3n)) ) { + if ( !(pir_u->log_u = u3_disk_init(pax_c, cb_u, c3y)) ) { c3_free(pir_u); return 0; } From 1556567b3333b3af4847753d239e0662e06f5c39 Mon Sep 17 00:00:00 2001 From: Matthew LeVan Date: Tue, 18 Jul 2023 12:03:44 -0400 Subject: [PATCH 069/271] disk/events: simplify code --- pkg/noun/events.c | 8 ++++---- pkg/vere/disk.c | 10 ++++------ 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/pkg/noun/events.c b/pkg/noun/events.c index 8b8d98993e..0294f522ab 100644 --- a/pkg/noun/events.c +++ b/pkg/noun/events.c @@ -1344,12 +1344,12 @@ c3_o u3e_backup(c3_c* pux_c, c3_c* pax_c, c3_o ovw_o) { // source image files from [pux_c] - u3e_image nux_u = { .nam_c = "north" }; - u3e_image sux_u = { .nam_c = "south" }; + u3e_image nux_u = { .nam_c = "north", .pgs_w = 0 }; + u3e_image sux_u = { .nam_c = "south", .pgs_w = 0 }; // destination image files to [pax_c] - u3e_image nax_u = { .nam_c = "north" }; - u3e_image sax_u = { .nam_c = "south" }; + u3e_image nax_u = { .nam_c = "north", .pgs_w = 0 }; + u3e_image sax_u = { .nam_c = "south", .pgs_w = 0 }; c3_i mod_i = O_RDWR | O_CREAT; diff --git a/pkg/vere/disk.c b/pkg/vere/disk.c index f87ce92154..e506b71153 100644 --- a/pkg/vere/disk.c +++ b/pkg/vere/disk.c @@ -1044,12 +1044,10 @@ u3_disk_init(c3_c* pax_c, u3_disk_cb cb_u, c3_o mig_o) } if ( c3y == u3_disk_need_migrate(log_u) ) { - if ( c3y == mig_o ) { - if ( c3n == u3_disk_migrate(log_u) ) { - fprintf(stderr, "disk: failed to migrate log\r\n"); - c3_free(log_u); - return 0; - } + if ( (c3y == mig_o) && (c3n == u3_disk_migrate(log_u)) ) { + fprintf(stderr, "disk: failed to migrate log\r\n"); + c3_free(log_u); + return 0; } else { fprintf(stderr, "disk: loading old format\r\n"); From f9a2d315c3354ca25d02493f48f5d67feed5d1ad Mon Sep 17 00:00:00 2001 From: Matthew LeVan Date: Wed, 26 Jul 2023 10:20:09 -0400 Subject: [PATCH 070/271] disk: add "migration starting" printf --- pkg/vere/disk.c | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/vere/disk.c b/pkg/vere/disk.c index e506b71153..f24c11e11b 100644 --- a/pkg/vere/disk.c +++ b/pkg/vere/disk.c @@ -1341,6 +1341,7 @@ u3_disk_migrate(u3_disk* log_u) * 5. delete old data.mdb and lock.mdb files (c3_unlink() calls) */ + fprintf(stderr, "disk: migrating disk to v%d format\r\n", U3D_VER1); // check if lock.mdb is readable in log directory c3_o luk_o = c3n; From d1661045fe7b77cb9041d66e33528657f855ea34 Mon Sep 17 00:00:00 2001 From: Matthew LeVan Date: Tue, 1 Aug 2023 12:09:44 -0400 Subject: [PATCH 071/271] vere: bump %lull kelvin to %322 --- pkg/vere/pier.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/vere/pier.c b/pkg/vere/pier.c index 1e39d14ef4..506a59a195 100644 --- a/pkg/vere/pier.c +++ b/pkg/vere/pier.c @@ -676,7 +676,7 @@ _pier_wyrd_fail(u3_pier* pir_u, u3_ovum* egg_u, u3_noun lud) // #define VERE_NAME "vere" #define VERE_ZUSE 412 -#define VERE_LULL 323 +#define VERE_LULL 322 /* _pier_wyrd_aver(): check for %wend effect and version downgrade. RETAIN */ From f06846d29daac1564a925355dae897853d68ea27 Mon Sep 17 00:00:00 2001 From: Matthew LeVan Date: Tue, 1 Aug 2023 12:10:18 -0400 Subject: [PATCH 072/271] vere: decrement zuse to %411 --- pkg/vere/pier.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/vere/pier.c b/pkg/vere/pier.c index 506a59a195..8e54dbb37e 100644 --- a/pkg/vere/pier.c +++ b/pkg/vere/pier.c @@ -675,7 +675,7 @@ _pier_wyrd_fail(u3_pier* pir_u, u3_ovum* egg_u, u3_noun lud) // XX organizing version constants // #define VERE_NAME "vere" -#define VERE_ZUSE 412 +#define VERE_ZUSE 411 #define VERE_LULL 322 /* _pier_wyrd_aver(): check for %wend effect and version downgrade. RETAIN From ea6f278bdf02b60d795489be04f35d471181769d Mon Sep 17 00:00:00 2001 From: Matthew LeVan Date: Wed, 16 Aug 2023 08:40:56 -0600 Subject: [PATCH 073/271] disk: move `u3C.wag_w` assignment before boot call --- pkg/vere/main.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/vere/main.c b/pkg/vere/main.c index b09ccdd346..cca3081a9b 100644 --- a/pkg/vere/main.c +++ b/pkg/vere/main.c @@ -1924,11 +1924,11 @@ _cw_meld(c3_i argc, c3_c* argv[]) exit(1); } + u3C.wag_w |= u3o_hashless; + u3_Host.eve_d = u3m_boot(u3_Host.dir_c, (size_t)1 << u3_Host.ops_u.lom_y); u3_disk* log_u = _cw_disk_init(u3_Host.dir_c); // XX s/b try_aquire lock - u3C.wag_w |= u3o_hashless; - u3a_print_memory(stderr, "urbit: meld: gained", u3u_meld()); u3m_save(); From 6ad0148f56b6208c147e8ea7f351ff3e71dead3d Mon Sep 17 00:00:00 2001 From: Pyry Kovanen Date: Wed, 23 Aug 2023 23:05:34 +0300 Subject: [PATCH 074/271] pier: decrement arvo kelvin --- pkg/vere/pier.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/vere/pier.c b/pkg/vere/pier.c index 8e54dbb37e..6bd0384d43 100644 --- a/pkg/vere/pier.c +++ b/pkg/vere/pier.c @@ -814,7 +814,7 @@ _pier_wyrd_card(u3_pier* pir_u) u3_nul); u3_noun kel = u3nl(u3nc(c3__zuse, VERE_ZUSE), // XX from both king and serf? u3nc(c3__lull, VERE_LULL), // XX from both king and serf? - u3nc(c3__arvo, 237), // XX from both king and serf? + u3nc(c3__arvo, 236), // XX from both king and serf? u3nc(c3__hoon, 139), // god_u->hon_y u3nc(c3__nock, 4), // god_u->noc_y u3_none); From b608007a6f648f352ecfb8320da3761b807e7e4d Mon Sep 17 00:00:00 2001 From: Ted Blackman Date: Fri, 1 Sep 2023 17:18:42 -0400 Subject: [PATCH 075/271] stun: WIP but compiles --- pkg/c3/motes.h | 1 + pkg/vere/io/ames.c | 215 ++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 212 insertions(+), 4 deletions(-) diff --git a/pkg/c3/motes.h b/pkg/c3/motes.h index a9fd4b701e..4f06492e37 100644 --- a/pkg/c3/motes.h +++ b/pkg/c3/motes.h @@ -996,6 +996,7 @@ # define c3__sard c3_s4('s','a','r','d') # define c3__sav c3_s3('s','a','v') # define c3__save c3_s4('s','a','v','e') +# define c3__saxo c3_s4('s','a','x','o') # define c3__scam c3_s4('s','c','a','m') # define c3__scan c3_s4('s','c','a','n') # define c3__scry c3_s4('s','c','r','y') diff --git a/pkg/vere/io/ames.c b/pkg/vere/io/ames.c index 185afad073..ba0774e1f1 100644 --- a/pkg/vere/io/ames.c +++ b/pkg/vere/io/ames.c @@ -17,6 +17,12 @@ #define QUEUE_MAX 30 // max number of packets in queue +typedef enum u3_stun_state { + STUN_OFF = 0, + STUN_TRYING = 1, + STUN_KEEPALIVE = 2, +} u3_stun_state; + /* u3_fine: fine networking */ typedef struct _u3_fine { @@ -45,6 +51,16 @@ c3_w imp_w[256]; // imperial IPs time_t imp_t[256]; // imperial IP timestamps c3_o imp_o[256]; // imperial print status + struct { // stun client state: + u3_stun_state sat_y; // formal state + c3_y tid_y[16]; // last transaction id + c3_d dad_d[2]; // sponsoring ship @p + u3_lane lan_u; // sponsoring ship IP and port + uv_timer_t tim_u; // keepalive timer handle + struct timeval las_u; // last sent date + struct timeval sar_u; // date we started trying to send + u3_lane sef_u; // our lane, if we know it + } sun_u; // struct { // config: c3_o net_o; // can send c3_o see_o; // can scry @@ -1173,6 +1189,193 @@ _fine_put_cache(u3_ames* sam_u, u3_noun pax, c3_w lop_w, u3_noun lis) } } +static void +_stun_stop(u3_ames* sam_u) +{ + switch ( sam_u->sun_u.sat_y ) { + case STUN_OFF: break; // ignore; already stopped + case STUN_TRYING: { + uv_timer_stop(&sam_u->sun_u.tim_u); + } break; + case STUN_KEEPALIVE: { + uv_timer_stop(&sam_u->sun_u.tim_u); + } break; + default: u3_assert(!"programmer error"); + } + sam_u->sun_u.sat_y = STUN_OFF; +} + +static void _stun_send_request(u3_ames*); // forward declaration + +static void +_stun_timer_cb(uv_timer_t* tim_u) +{ + u3_ames* sam_u = (u3_ames*)(tim_u->data); + switch ( sam_u->sun_u.sat_y ) { + case STUN_OFF: { + // ignore; stray timer (although this shouldn't happen) + u3l_log("stun: stray timer STUN_OFF"); + } break; + case STUN_KEEPALIVE: { + sam_u->sun_u.sat_y = STUN_TRYING; + _stun_send_request(sam_u); + } break; + case STUN_TRYING: { + c3_d gap_d; + { + struct timeval tim_tv; + gettimeofday(&tim_tv, 0); + u3_noun now = u3_time_in_tv(&tim_tv); + u3_noun den = u3_time_in_tv(&sam_u->sun_u.sar_u); + gap_d = u3_time_gap_ms(den, now); + } + if ( gap_d >= (5 * 1000) ) { + _stun_stop(sam_u); + // TODO inject event into arvo to ping repeatedly + } + else { + sam_u->sun_u.tim_u.data = sam_u; + uv_timer_start(&sam_u->sun_u.tim_u, _stun_timer_cb, 1*1000, 0); + } + } break; + default: u3_assert(!"programmer error"); + } +} + +typedef struct _u3_stun_send { + uv_udp_send_t req_u; // uv udp request handle + u3_ames* sam_u; // backpointer to driver state +} u3_stun_send; + +static void +_stun_send_cb(uv_udp_send_t *req_u, c3_i sas_i) +{ + u3_stun_send* snd_u = (u3_stun_send*)req_u; + if ( sas_i ) { + // TODO take error handling action + } + else { + u3_ames* sam_u = snd_u->sam_u; + gettimeofday(&sam_u->sun_u.las_u, 0); // overwrite last sent date + } +} + +static void +_stun_on_response(u3_ames* sam_u) // TODO read arg +{ + u3_stun_state old_y = sam_u->sun_u.sat_y; + switch ( sam_u->sun_u.sat_y ) { + case STUN_OFF: break; // ignore; stray response + case STUN_KEEPALIVE: break; // ignore; duplicate response + case STUN_TRYING: { + sam_u->sun_u.sat_y = STUN_KEEPALIVE; + sam_u->sun_u.tid_y[1]++; + uv_timer_stop(&sam_u->sun_u.tim_u); + sam_u->sun_u.tim_u.data = sam_u; + uv_timer_start(&sam_u->sun_u.tim_u, _stun_timer_cb, 25*1000, 0); + } break; + default: assert("programmer error"); + } +} + +static void +_stun_send_request(u3_ames* sam_u) +{ + u3_assert( STUN_OFF != sam_u->sun_u.sat_y ); + + struct sockaddr_in add_u; + memset(&add_u, 0, sizeof(add_u)); + add_u.sin_family = AF_INET; + add_u.sin_addr.s_addr = htonl(sam_u->sun_u.lan_u.pip_w); + add_u.sin_port = htons(sam_u->sun_u.lan_u.por_s); + + c3_y buf_y[20] = {0}; + buf_y[1] = 0x1; // message type "binding request" + memcpy(buf_y + 4, sam_u->sun_u.tid_y, 16); // TODO convert byte order? + + uv_buf_t buf_u = uv_buf_init((c3_c*)buf_y, 20); + u3_stun_send* snd_u = c3_calloc(sizeof(*snd_u)); + snd_u->sam_u = sam_u; + c3_i sas_i = uv_udp_send( + (uv_udp_send_t*)snd_u, &sam_u->wax_u, &buf_u, 1, + (const struct sockaddr*)&add_u, _stun_send_cb + ); + + if ( sas_i ) { + u3l_log("stun: send fail_sync: %s", uv_strerror(sas_i)); + // TODO take error handling actions + } +} + +static void +_stun_czar_cb(uv_getaddrinfo_t* adr_u, + c3_i sas_i, + struct addrinfo* aif_u) +{ + u3_ames* sam_u; // TODO get sam_u from args + gettimeofday(&sam_u->sun_u.sar_u, 0); // set start time to now + _stun_send_request(sam_u); + uv_timer_start(&sam_u->sun_u.tim_u, _stun_timer_cb, 1, 0); + +} + +static void +_stun_czar(u3_ames* sam_u) +{ + // TODO rip off _ames_czar() +} + +static void +_stun_start(u3_ames* sam_u, u3_noun dad) +{ + sam_u->sun_u.sat_y = STUN_TRYING; + u3r_chubs(0, 2, sam_u->sun_u.dad_d, dad); + u3z(dad); + + // initialize STUN transaction id to mug of now + { + u3_noun mug; + struct timeval tim_u; + gettimeofday(&tim_u, 0); + + mug = u3r_mug(u3_time_in_tv(&tim_u)); + memcpy(sam_u->sun_u.tid_y, (c3_y*)&mug, 4); + u3z(mug); + } + + _stun_czar(sam_u); +} + +static c3_o +_ames_is_czar(u3_noun who) +{ + u3_noun rac = u3do("clan:title", u3k(who)); + c3_o zar = ( c3y == (c3__czar == rac) ); + u3z(rac); + return zar; +} + +/* _ames_ef_saxo(): handle sponsorship chain notification +*/ +static void +_ames_ef_saxo(u3_ames* sam_u, u3_noun zad) +{ + u3_noun daz, dad; + + daz = u3qb_flop(zad); + if ( u3_nul == daz ) { + u3l_log("ames: empty sponsorship chain"); + return; + } + + dad = u3k(u3h(daz)); + if ( c3y == _ames_is_czar(dad) ) { + _stun_stop(sam_u); + _stun_start(sam_u, u3k(dad)); + } + + u3z(zad); u3z(daz); u3z(dad); +} /* _ames_ef_send(): send packet to network (v4). */ @@ -2064,10 +2267,11 @@ _ames_io_start(u3_ames* sam_u) { c3_s por_s = sam_u->pir_u->por_s; u3_noun who = u3i_chubs(2, sam_u->pir_u->who_d); - u3_noun rac = u3do("clan:title", u3k(who)); + c3_o zar_o = _ames_is_czar(who); c3_i ret_i; - if ( c3__czar == rac ) { + + if ( c3y == zar_o ) { c3_y num_y = (c3_y)sam_u->pir_u->who_d[0]; c3_s zar_s = _ames_czar_port(num_y); @@ -2097,7 +2301,7 @@ _ames_io_start(u3_ames* sam_u) { u3l_log("ames: bind: %s", uv_strerror(ret_i)); - if ( (c3__czar == rac) && + if ( (c3y == zar_o) && (UV_EADDRINUSE == ret_i) ) { u3l_log(" ...perhaps you've got two copies of vere running?"); @@ -2124,7 +2328,6 @@ _ames_io_start(u3_ames* sam_u) uv_udp_recv_start(&sam_u->wax_u, _ames_alloc, _ames_recv_cb); sam_u->car_u.liv_o = c3y; - u3z(rac); u3z(who); } @@ -2273,6 +2476,10 @@ _ames_kick_newt(u3_ames* sam_u, u3_noun tag, u3_noun dat) _ames_ef_turf(sam_u, u3k(dat)); ret_o = c3y; } break; + + case c3__saxo: { + _ames_ef_saxo(sam_u, u3k(dat)); + } break; } u3z(tag); u3z(dat); From 94d4bb0b49bf699bc9965becaacfb9d861ef74f6 Mon Sep 17 00:00:00 2001 From: Matthew LeVan Date: Sat, 19 Aug 2023 18:20:13 -0400 Subject: [PATCH 076/271] build: upgrade `zlib` to v1.3 --- WORKSPACE.bazel | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/WORKSPACE.bazel b/WORKSPACE.bazel index 56a347ed6d..2cad948ace 100644 --- a/WORKSPACE.bazel +++ b/WORKSPACE.bazel @@ -358,10 +358,10 @@ versioned_http_archive( versioned_http_archive( name = "zlib", build_file = "//bazel/third_party/zlib:zlib.BUILD", - sha256 = "b3a24de97a8fdbc835b9833169501030b8977031bcb54b3b3ac13740f846ab30", + sha256 = "ff0ba4c292013dbc27530b3a81e1f9a813cd39de01ca5e0f8bf355702efa593e", strip_prefix = "zlib-{version}", url = "https://www.zlib.net/zlib-{version}.tar.gz", - version = "1.2.13", + version = "1.3", ) # From 704a619abab0e8b7d18477fb6804d93434d4d3c0 Mon Sep 17 00:00:00 2001 From: pkova Date: Tue, 29 Aug 2023 19:35:06 +0300 Subject: [PATCH 077/271] mdns: add mdns for local network routing --- WORKSPACE.bazel | 28 + bazel/third_party/avahi/BUILD.bazel | 0 bazel/third_party/avahi/avahi.BUILD | 30 + bazel/third_party/dbus/BUILD.bazel | 0 bazel/third_party/dbus/dbus.BUILD | 21 + bazel/third_party/expat/BUILD.bazel | 0 bazel/third_party/expat/expat.BUILD | 20 + pkg/c3/motes.h | 1 + pkg/vere/BUILD.bazel | 14 +- pkg/vere/dns_sd.h | 1722 +++++++++++++++++++++++++++ pkg/vere/io/ames.c | 50 + pkg/vere/mdns.c | 226 ++++ pkg/vere/vere.h | 12 + 13 files changed, 2122 insertions(+), 2 deletions(-) create mode 100644 bazel/third_party/avahi/BUILD.bazel create mode 100644 bazel/third_party/avahi/avahi.BUILD create mode 100644 bazel/third_party/dbus/BUILD.bazel create mode 100644 bazel/third_party/dbus/dbus.BUILD create mode 100644 bazel/third_party/expat/BUILD.bazel create mode 100644 bazel/third_party/expat/expat.BUILD create mode 100644 pkg/vere/dns_sd.h create mode 100644 pkg/vere/mdns.c diff --git a/WORKSPACE.bazel b/WORKSPACE.bazel index 56a347ed6d..d1dad3d96a 100644 --- a/WORKSPACE.bazel +++ b/WORKSPACE.bazel @@ -130,6 +130,15 @@ versioned_http_archive( version = "a4c1e3f7138c2e577376beb99f964cf71e1c8b1b", ) +versioned_http_archive( + name = "avahi", + build_file = "//bazel/third_party/avahi:avahi.BUILD", + sha256 = "060309d7a333d38d951bc27598c677af1796934dbd98e1024e7ad8de798fedda", + strip_prefix = "avahi-{version}", + url = "https://github.com/lathiat/avahi/releases/download/v{version}/avahi-{version}.tar.gz", + version = "0.8", +) + versioned_http_archive( name = "bazel_gazelle", sha256 = "efbbba6ac1a4fd342d5122cbdfdb82aeb2cf2862e35022c752eaddffada7c3f3", @@ -154,6 +163,15 @@ versioned_http_archive( version = "7.85.0", ) +versioned_http_archive( + name = "dbus", + build_file = "//bazel/third_party/dbus:dbus.BUILD", + sha256 = "a6bd5bac5cf19f0c3c594bdae2565a095696980a683a0ef37cb6212e093bde35", + strip_prefix = "dbus-{version}", + url = "https://dbus.freedesktop.org/releases/dbus/dbus-{version}.tar.xz", + version = "1.14.8", +) + versioned_http_archive( name = "ed25519", build_file = "//bazel/third_party/ed25519:ed25519.BUILD", @@ -163,6 +181,16 @@ versioned_http_archive( version = "7fa6712ef5d581a6981ec2b08ee623314cd1d1c4", ) +versioned_http_archive( + name = "expat", + build_file = "//bazel/third_party/expat:expat.BUILD", + strip_prefix = "expat-{version}", + sha256 = "ef2420f0232c087801abf705e89ae65f6257df6b7931d37846a193ef2e8cdcbe", + # TODO: fix the R_2_5_0 nonsense + url = "https://github.com/libexpat/libexpat/releases/download/R_2_5_0/expat-{version}.tar.xz", + version = "2.5.0", +) + versioned_http_archive( name = "gmp", build_file = "//bazel/third_party/gmp:gmp.BUILD", diff --git a/bazel/third_party/avahi/BUILD.bazel b/bazel/third_party/avahi/BUILD.bazel new file mode 100644 index 0000000000..e69de29bb2 diff --git a/bazel/third_party/avahi/avahi.BUILD b/bazel/third_party/avahi/avahi.BUILD new file mode 100644 index 0000000000..84d27171c2 --- /dev/null +++ b/bazel/third_party/avahi/avahi.BUILD @@ -0,0 +1,30 @@ +load("@rules_foreign_cc//foreign_cc:defs.bzl", "configure_make") + +filegroup( + name = "all", + srcs = glob(["**"]), +) + +cc_library( + name = "dns-sd", + hdrs = ["dns_sd.h"], + visibility = ["//visibility:public"], +) + +configure_make( + name = "avahi", + args = select({ + "@platforms//os:macos": ["--jobs=`sysctl -n hw.logicalcpu`"], + "//conditions:default": ["--jobs=`nproc`"], + }), + configure_options = ["--with-dbus-system-address='unix:path=/var/run/dbus/system_bus_socket' --with-xml=none --disable-libevent --disable-glib --disable-gobject --disable-gdbm --disable-qt3 --disable-qt4 --disable-qt5 --disable-gtk --disable-gtk3 --disable-mono --disable-monodoc --disable-python --disable-libdaemon --enable-compat-libdns_sd --disable-rpath"], + lib_source = ":all", + # out_include_dir = "avahi-compat-libdns_sd", + deps = ["@dbus"], + configure_in_place = True, + autogen = True, + autoconf = True, + autogen_command = "bootstrap.sh", + out_static_libs = ["libdns_sd.a", "libavahi-client.a", "libavahi-common.a"], + visibility = ["//visibility:public"], +) diff --git a/bazel/third_party/dbus/BUILD.bazel b/bazel/third_party/dbus/BUILD.bazel new file mode 100644 index 0000000000..e69de29bb2 diff --git a/bazel/third_party/dbus/dbus.BUILD b/bazel/third_party/dbus/dbus.BUILD new file mode 100644 index 0000000000..ac55b84103 --- /dev/null +++ b/bazel/third_party/dbus/dbus.BUILD @@ -0,0 +1,21 @@ +load("@rules_foreign_cc//foreign_cc:defs.bzl", "configure_make") + +filegroup( + name = "all", + srcs = glob(["**"]), +) + +configure_make( + name = "dbus", + lib_name = "libdbus-1", + args = select({ + "@platforms//os:macos": ["--jobs=`sysctl -n hw.logicalcpu`"], + "//conditions:default": ["--jobs=`nproc`"], + }), + copts = ["-O3"], + configure_options = ["--disable-selinux"], + lib_source = ":all", + configure_in_place = True, + deps = ["@expat"], + visibility = ["//visibility:public"], +) diff --git a/bazel/third_party/expat/BUILD.bazel b/bazel/third_party/expat/BUILD.bazel new file mode 100644 index 0000000000..e69de29bb2 diff --git a/bazel/third_party/expat/expat.BUILD b/bazel/third_party/expat/expat.BUILD new file mode 100644 index 0000000000..10e536a620 --- /dev/null +++ b/bazel/third_party/expat/expat.BUILD @@ -0,0 +1,20 @@ +load("@rules_foreign_cc//foreign_cc:defs.bzl", "configure_make") + +filegroup( + name = "all", + srcs = glob(["**"]), +) + +configure_make( + name = "expat", + args = select({ + "@platforms//os:macos": ["--jobs=`sysctl -n hw.logicalcpu`"], + "//conditions:default": ["--jobs=`nproc`"], + }), + # configure_options = ["--with-xml=none --disable-libevent --disable-glib --disable-gobject --disable-gdbm --disable-qt3 --disable-qt4 --disable-qt5 --disable-gtk --disable-gtk3 --disable-mono --disable-monodoc --disable-python --enable-compat-libdns_sd"], + copts = ["-O3"], + lib_source = ":all", + # deps = ["@dbus"], + out_static_libs = ["libexpat.a"], + visibility = ["//visibility:public"], +) diff --git a/pkg/c3/motes.h b/pkg/c3/motes.h index 0000d5c593..db17834057 100644 --- a/pkg/c3/motes.h +++ b/pkg/c3/motes.h @@ -308,6 +308,7 @@ # define c3__deep c3_s4('d','e','e','p') # define c3__defn c3_s4('d','e','f','n') # define c3__del c3_s3('d','e','l') +# define c3__dear c3_s4('d','e','a','r') # define c3__delc c3_s4('d','e','l','c') # define c3__delt c3_s4('d','e','l','t') # define c3__dept c3_s4('d','e','p','t') diff --git a/pkg/vere/BUILD.bazel b/pkg/vere/BUILD.bazel index 76f8b47eca..249fc42e42 100644 --- a/pkg/vere/BUILD.bazel +++ b/pkg/vere/BUILD.bazel @@ -119,7 +119,12 @@ vere_library( hdrs = [ "db/lmdb.h", "vere.h", - ], + ] + select({ + "@platforms//os:macos": [], + "@platforms//os:linux": [ + "dns_sd.h", + ], + }), includes = ["."], linkstatic = True, visibility = ["//pkg:__subpackages__"], @@ -133,7 +138,12 @@ vere_library( "@lmdb", "@openssl", "@uv", - ], + ] + select({ + "@platforms//os:macos": [], + "@platforms//os:linux": [ + "@avahi", + ], + }) ) # diff --git a/pkg/vere/dns_sd.h b/pkg/vere/dns_sd.h new file mode 100644 index 0000000000..66e494e402 --- /dev/null +++ b/pkg/vere/dns_sd.h @@ -0,0 +1,1722 @@ +/* -*- Mode: C; tab-width: 4 -*- + * + * Copyright (c) 2003-2004, Apple Computer, Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * 3. Neither the name of Apple Computer, Inc. ("Apple") nor the names of its + * contributors may be used to endorse or promote products derived from this + * software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY APPLE AND ITS CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL APPLE OR ITS CONTRIBUTORS BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _DNS_SD_H +#define _DNS_SD_H + +#ifdef __cplusplus + extern "C" { +#endif + +/* standard calling convention under Win32 is __stdcall */ +/* Note: When compiling Intel EFI (Extensible Firmware Interface) under MS Visual Studio, the */ +/* _WIN32 symbol is defined by the compiler even though it's NOT compiling code for Windows32 */ +#if defined(_WIN32) && !defined(EFI32) && !defined(EFI64) +#define DNSSD_API __stdcall +#else +#define DNSSD_API +#endif + +/* stdint.h does not exist on FreeBSD 4.x; its types are defined in sys/types.h instead */ +#if defined(__FreeBSD__) && (__FreeBSD__ < 5) +#include + +/* Likewise, on Sun, standard integer types are in sys/types.h */ +#elif defined(__sun__) +#include + +/* EFI does not have stdint.h, or anything else equivalent */ +#elif defined(EFI32) || defined(EFI64) +typedef UINT8 uint8_t; +typedef INT8 int8_t; +typedef UINT16 uint16_t; +typedef INT16 int16_t; +typedef UINT32 uint32_t; +typedef INT32 int32_t; + +/* Windows has its own differences */ +#elif defined(_WIN32) +#include +#define _UNUSED +#define bzero(a, b) memset(a, 0, b) +#ifndef _MSL_STDINT_H +typedef UINT8 uint8_t; +typedef INT8 int8_t; +typedef UINT16 uint16_t; +typedef INT16 int16_t; +typedef UINT32 uint32_t; +typedef INT32 int32_t; +#endif + +/* All other Posix platforms use stdint.h */ +#else +#include +#endif + +/* DNSServiceRef, DNSRecordRef + * + * Opaque internal data types. + * Note: client is responsible for serializing access to these structures if + * they are shared between concurrent threads. + */ + +typedef struct _DNSServiceRef_t *DNSServiceRef; +typedef struct _DNSRecordRef_t *DNSRecordRef; + +/* General flags used in functions defined below */ +enum + { + kDNSServiceFlagsMoreComing = 0x1, + /* MoreComing indicates to a callback that at least one more result is + * queued and will be delivered following immediately after this one. + * Applications should not update their UI to display browse + * results when the MoreComing flag is set, because this would + * result in a great deal of ugly flickering on the screen. + * Applications should instead wait until until MoreComing is not set, + * and then update their UI. + * When MoreComing is not set, that doesn't mean there will be no more + * answers EVER, just that there are no more answers immediately + * available right now at this instant. If more answers become available + * in the future they will be delivered as usual. + */ + + kDNSServiceFlagsAdd = 0x2, + kDNSServiceFlagsDefault = 0x4, + /* Flags for domain enumeration and browse/query reply callbacks. + * "Default" applies only to enumeration and is only valid in + * conjuction with "Add". An enumeration callback with the "Add" + * flag NOT set indicates a "Remove", i.e. the domain is no longer + * valid. + */ + + kDNSServiceFlagsNoAutoRename = 0x8, + /* Flag for specifying renaming behavior on name conflict when registering + * non-shared records. By default, name conflicts are automatically handled + * by renaming the service. NoAutoRename overrides this behavior - with this + * flag set, name conflicts will result in a callback. The NoAutorename flag + * is only valid if a name is explicitly specified when registering a service + * (i.e. the default name is not used.) + */ + + kDNSServiceFlagsShared = 0x10, + kDNSServiceFlagsUnique = 0x20, + /* Flag for registering individual records on a connected + * DNSServiceRef. Shared indicates that there may be multiple records + * with this name on the network (e.g. PTR records). Unique indicates that the + * record's name is to be unique on the network (e.g. SRV records). + */ + + kDNSServiceFlagsBrowseDomains = 0x40, + kDNSServiceFlagsRegistrationDomains = 0x80, + /* Flags for specifying domain enumeration type in DNSServiceEnumerateDomains. + * BrowseDomains enumerates domains recommended for browsing, RegistrationDomains + * enumerates domains recommended for registration. + */ + + kDNSServiceFlagsLongLivedQuery = 0x100, + /* Flag for creating a long-lived unicast query for the DNSServiceQueryRecord call. */ + + kDNSServiceFlagsAllowRemoteQuery = 0x200, + /* Flag for creating a record for which we will answer remote queries + * (queries from hosts more than one hop away; hosts not directly connected to the local link). + */ + + kDNSServiceFlagsForceMulticast = 0x400, + /* Flag for signifying that a query or registration should be performed exclusively via multicast DNS, + * even for a name in a domain (e.g. foo.apple.com.) that would normally imply unicast DNS. + */ + + kDNSServiceFlagsReturnCNAME = 0x800 + /* Flag for returning CNAME records in the DNSServiceQueryRecord call. CNAME records are + * normally followed without indicating to the client that there was a CNAME record. + */ + }; + +/* + * The values for DNS Classes and Types are listed in RFC 1035, and are available + * on every OS in its DNS header file. Unfortunately every OS does not have the + * same header file containing DNS Class and Type constants, and the names of + * the constants are not consistent. For example, BIND 8 uses "T_A", + * BIND 9 uses "ns_t_a", Windows uses "DNS_TYPE_A", etc. + * For this reason, these constants are also listed here, so that code using + * the DNS-SD programming APIs can use these constants, so that the same code + * can compile on all our supported platforms. + */ + +enum + { + kDNSServiceClass_IN = 1 /* Internet */ + }; + +enum + { + kDNSServiceType_A = 1, /* Host address. */ + kDNSServiceType_NS = 2, /* Authoritative server. */ + kDNSServiceType_MD = 3, /* Mail destination. */ + kDNSServiceType_MF = 4, /* Mail forwarder. */ + kDNSServiceType_CNAME = 5, /* Canonical name. */ + kDNSServiceType_SOA = 6, /* Start of authority zone. */ + kDNSServiceType_MB = 7, /* Mailbox domain name. */ + kDNSServiceType_MG = 8, /* Mail group member. */ + kDNSServiceType_MR = 9, /* Mail rename name. */ + kDNSServiceType_NULL = 10, /* Null resource record. */ + kDNSServiceType_WKS = 11, /* Well known service. */ + kDNSServiceType_PTR = 12, /* Domain name pointer. */ + kDNSServiceType_HINFO = 13, /* Host information. */ + kDNSServiceType_MINFO = 14, /* Mailbox information. */ + kDNSServiceType_MX = 15, /* Mail routing information. */ + kDNSServiceType_TXT = 16, /* One or more text strings. */ + kDNSServiceType_RP = 17, /* Responsible person. */ + kDNSServiceType_AFSDB = 18, /* AFS cell database. */ + kDNSServiceType_X25 = 19, /* X_25 calling address. */ + kDNSServiceType_ISDN = 20, /* ISDN calling address. */ + kDNSServiceType_RT = 21, /* Router. */ + kDNSServiceType_NSAP = 22, /* NSAP address. */ + kDNSServiceType_NSAP_PTR = 23, /* Reverse NSAP lookup (deprecated). */ + kDNSServiceType_SIG = 24, /* Security signature. */ + kDNSServiceType_KEY = 25, /* Security key. */ + kDNSServiceType_PX = 26, /* X.400 mail mapping. */ + kDNSServiceType_GPOS = 27, /* Geographical position (withdrawn). */ + kDNSServiceType_AAAA = 28, /* IPv6 Address. */ + kDNSServiceType_LOC = 29, /* Location Information. */ + kDNSServiceType_NXT = 30, /* Next domain (security). */ + kDNSServiceType_EID = 31, /* Endpoint identifier. */ + kDNSServiceType_NIMLOC = 32, /* Nimrod Locator. */ + kDNSServiceType_SRV = 33, /* Server Selection. */ + kDNSServiceType_ATMA = 34, /* ATM Address */ + kDNSServiceType_NAPTR = 35, /* Naming Authority PoinTeR */ + kDNSServiceType_KX = 36, /* Key Exchange */ + kDNSServiceType_CERT = 37, /* Certification record */ + kDNSServiceType_A6 = 38, /* IPv6 Address (deprecated) */ + kDNSServiceType_DNAME = 39, /* Non-terminal DNAME (for IPv6) */ + kDNSServiceType_SINK = 40, /* Kitchen sink (experimentatl) */ + kDNSServiceType_OPT = 41, /* EDNS0 option (meta-RR) */ + kDNSServiceType_TKEY = 249, /* Transaction key */ + kDNSServiceType_TSIG = 250, /* Transaction signature. */ + kDNSServiceType_IXFR = 251, /* Incremental zone transfer. */ + kDNSServiceType_AXFR = 252, /* Transfer zone of authority. */ + kDNSServiceType_MAILB = 253, /* Transfer mailbox records. */ + kDNSServiceType_MAILA = 254, /* Transfer mail agent records. */ + kDNSServiceType_ANY = 255 /* Wildcard match. */ + }; + + +/* possible error code values */ +enum + { + kDNSServiceErr_NoError = 0, + kDNSServiceErr_Unknown = -65537, /* 0xFFFE FFFF */ + kDNSServiceErr_NoSuchName = -65538, + kDNSServiceErr_NoMemory = -65539, + kDNSServiceErr_BadParam = -65540, + kDNSServiceErr_BadReference = -65541, + kDNSServiceErr_BadState = -65542, + kDNSServiceErr_BadFlags = -65543, + kDNSServiceErr_Unsupported = -65544, + kDNSServiceErr_NotInitialized = -65545, + kDNSServiceErr_AlreadyRegistered = -65547, + kDNSServiceErr_NameConflict = -65548, + kDNSServiceErr_Invalid = -65549, + kDNSServiceErr_Firewall = -65550, + kDNSServiceErr_Incompatible = -65551, /* client library incompatible with daemon */ + kDNSServiceErr_BadInterfaceIndex = -65552, + kDNSServiceErr_Refused = -65553, + kDNSServiceErr_NoSuchRecord = -65554, + kDNSServiceErr_NoAuth = -65555, + kDNSServiceErr_NoSuchKey = -65556, + kDNSServiceErr_NATTraversal = -65557, + kDNSServiceErr_DoubleNAT = -65558, + kDNSServiceErr_BadTime = -65559 + /* mDNS Error codes are in the range + * FFFE FF00 (-65792) to FFFE FFFF (-65537) */ + }; + + +/* Maximum length, in bytes, of a service name represented as a */ +/* literal C-String, including the terminating NULL at the end. */ + +#define kDNSServiceMaxServiceName 64 + +/* Maximum length, in bytes, of a domain name represented as an *escaped* C-String */ +/* including the final trailing dot, and the C-String terminating NULL at the end. */ + +#define kDNSServiceMaxDomainName 1005 + +/* + * Notes on DNS Name Escaping + * -- or -- + * "Why is kDNSServiceMaxDomainName 1005, when the maximum legal domain name is 255 bytes?" + * + * All strings used in DNS-SD are UTF-8 strings. + * With few exceptions, most are also escaped using standard DNS escaping rules: + * + * '\\' represents a single literal '\' in the name + * '\.' represents a single literal '.' in the name + * '\ddd', where ddd is a three-digit decimal value from 000 to 255, + * represents a single literal byte with that value. + * A bare unescaped '.' is a label separator, marking a boundary between domain and subdomain. + * + * The exceptions, that do not use escaping, are the routines where the full + * DNS name of a resource is broken, for convenience, into servicename/regtype/domain. + * In these routines, the "servicename" is NOT escaped. It does not need to be, since + * it is, by definition, just a single literal string. Any characters in that string + * represent exactly what they are. The "regtype" portion is, technically speaking, + * escaped, but since legal regtypes are only allowed to contain letters, digits, + * and hyphens, there is nothing to escape, so the issue is moot. The "domain" + * portion is also escaped, though most domains in use on the public Internet + * today, like regtypes, don't contain any characters that need to be escaped. + * As DNS-SD becomes more popular, rich-text domains for service discovery will + * become common, so software should be written to cope with domains with escaping. + * + * The servicename may be up to 63 bytes of UTF-8 text (not counting the C-String + * terminating NULL at the end). The regtype is of the form _service._tcp or + * _service._udp, where the "service" part is 1-14 characters, which may be + * letters, digits, or hyphens. The domain part of the three-part name may be + * any legal domain, providing that the resulting servicename+regtype+domain + * name does not exceed 255 bytes. + * + * For most software, these issues are transparent. When browsing, the discovered + * servicenames should simply be displayed as-is. When resolving, the discovered + * servicename/regtype/domain are simply passed unchanged to DNSServiceResolve(). + * When a DNSServiceResolve() succeeds, the returned fullname is already in + * the correct format to pass to standard system DNS APIs such as res_query(). + * For converting from servicename/regtype/domain to a single properly-escaped + * full DNS name, the helper function DNSServiceConstructFullName() is provided. + * + * The following (highly contrived) example illustrates the escaping process. + * Suppose you have an service called "Dr. Smith\Dr. Johnson", of type "_ftp._tcp" + * in subdomain "4th. Floor" of subdomain "Building 2" of domain "apple.com." + * The full (escaped) DNS name of this service's SRV record would be: + * Dr\.\032Smith\\Dr\.\032Johnson._ftp._tcp.4th\.\032Floor.Building\0322.apple.com. + */ + + +/* + * Constants for specifying an interface index + * + * Specific interface indexes are identified via a 32-bit unsigned integer returned + * by the if_nametoindex() family of calls. + * + * If the client passes 0 for interface index, that means "do the right thing", + * which (at present) means, "if the name is in an mDNS local multicast domain + * (e.g. 'local.', '254.169.in-addr.arpa.', '{8,9,A,B}.E.F.ip6.arpa.') then multicast + * on all applicable interfaces, otherwise send via unicast to the appropriate + * DNS server." Normally, most clients will use 0 for interface index to + * automatically get the default sensible behaviour. + * + * If the client passes a positive interface index, then for multicast names that + * indicates to do the operation only on that one interface. For unicast names the + * interface index is ignored unless kDNSServiceFlagsForceMulticast is also set. + * + * If the client passes kDNSServiceInterfaceIndexLocalOnly when registering + * a service, then that service will be found *only* by other local clients + * on the same machine that are browsing using kDNSServiceInterfaceIndexLocalOnly + * or kDNSServiceInterfaceIndexAny. + * If a client has a 'private' service, accessible only to other processes + * running on the same machine, this allows the client to advertise that service + * in a way such that it does not inadvertently appear in service lists on + * all the other machines on the network. + * + * If the client passes kDNSServiceInterfaceIndexLocalOnly when browsing + * then it will find *all* records registered on that same local machine. + * Clients explicitly wishing to discover *only* LocalOnly services can + * accomplish this by inspecting the interfaceIndex of each service reported + * to their DNSServiceBrowseReply() callback function, and discarding those + * where the interface index is not kDNSServiceInterfaceIndexLocalOnly. + */ + +#define kDNSServiceInterfaceIndexAny 0 +#define kDNSServiceInterfaceIndexLocalOnly ( (uint32_t) -1 ) + + +typedef uint32_t DNSServiceFlags; +typedef int32_t DNSServiceErrorType; + + +/********************************************************************************************* + * + * Unix Domain Socket access, DNSServiceRef deallocation, and data processing functions + * + *********************************************************************************************/ + + +/* DNSServiceRefSockFD() + * + * Access underlying Unix domain socket for an initialized DNSServiceRef. + * The DNS Service Discovery implmementation uses this socket to communicate between + * the client and the mDNSResponder daemon. The application MUST NOT directly read from + * or write to this socket. Access to the socket is provided so that it can be used as a + * run loop source, or in a select() loop: when data is available for reading on the socket, + * DNSServiceProcessResult() should be called, which will extract the daemon's reply from + * the socket, and pass it to the appropriate application callback. By using a run loop or + * select(), results from the daemon can be processed asynchronously. Without using these + * constructs, DNSServiceProcessResult() will block until the response from the daemon arrives. + * The client is responsible for ensuring that the data on the socket is processed in a timely + * fashion - the daemon may terminate its connection with a client that does not clear its + * socket buffer. + * + * sdRef: A DNSServiceRef initialized by any of the DNSService calls. + * + * return value: The DNSServiceRef's underlying socket descriptor, or -1 on + * error. + */ + +int DNSSD_API DNSServiceRefSockFD(DNSServiceRef sdRef); + + +/* DNSServiceProcessResult() + * + * Read a reply from the daemon, calling the appropriate application callback. This call will + * block until the daemon's response is received. Use DNSServiceRefSockFD() in + * conjunction with a run loop or select() to determine the presence of a response from the + * server before calling this function to process the reply without blocking. Call this function + * at any point if it is acceptable to block until the daemon's response arrives. Note that the + * client is responsible for ensuring that DNSServiceProcessResult() is called whenever there is + * a reply from the daemon - the daemon may terminate its connection with a client that does not + * process the daemon's responses. + * + * sdRef: A DNSServiceRef initialized by any of the DNSService calls + * that take a callback parameter. + * + * return value: Returns kDNSServiceErr_NoError on success, otherwise returns + * an error code indicating the specific failure that occurred. + */ + +DNSServiceErrorType DNSSD_API DNSServiceProcessResult(DNSServiceRef sdRef); + + +/* DNSServiceRefDeallocate() + * + * Terminate a connection with the daemon and free memory associated with the DNSServiceRef. + * Any services or records registered with this DNSServiceRef will be deregistered. Any + * Browse, Resolve, or Query operations called with this reference will be terminated. + * + * Note: If the reference's underlying socket is used in a run loop or select() call, it should + * be removed BEFORE DNSServiceRefDeallocate() is called, as this function closes the reference's + * socket. + * + * Note: If the reference was initialized with DNSServiceCreateConnection(), any DNSRecordRefs + * created via this reference will be invalidated by this call - the resource records are + * deregistered, and their DNSRecordRefs may not be used in subsequent functions. Similarly, + * if the reference was initialized with DNSServiceRegister, and an extra resource record was + * added to the service via DNSServiceAddRecord(), the DNSRecordRef created by the Add() call + * is invalidated when this function is called - the DNSRecordRef may not be used in subsequent + * functions. + * + * Note: This call is to be used only with the DNSServiceRef defined by this API. It is + * not compatible with dns_service_discovery_ref objects defined in the legacy Mach-based + * DNSServiceDiscovery.h API. + * + * sdRef: A DNSServiceRef initialized by any of the DNSService calls. + * + */ + +void DNSSD_API DNSServiceRefDeallocate(DNSServiceRef sdRef); + + +/********************************************************************************************* + * + * Domain Enumeration + * + *********************************************************************************************/ + +/* DNSServiceEnumerateDomains() + * + * Asynchronously enumerate domains available for browsing and registration. + * + * The enumeration MUST be cancelled via DNSServiceRefDeallocate() when no more domains + * are to be found. + * + * Note that the names returned are (like all of DNS-SD) UTF-8 strings, + * and are escaped using standard DNS escaping rules. + * (See "Notes on DNS Name Escaping" earlier in this file for more details.) + * A graphical browser displaying a hierarchical tree-structured view should cut + * the names at the bare dots to yield individual labels, then de-escape each + * label according to the escaping rules, and then display the resulting UTF-8 text. + * + * DNSServiceDomainEnumReply Callback Parameters: + * + * sdRef: The DNSServiceRef initialized by DNSServiceEnumerateDomains(). + * + * flags: Possible values are: + * kDNSServiceFlagsMoreComing + * kDNSServiceFlagsAdd + * kDNSServiceFlagsDefault + * + * interfaceIndex: Specifies the interface on which the domain exists. (The index for a given + * interface is determined via the if_nametoindex() family of calls.) + * + * errorCode: Will be kDNSServiceErr_NoError (0) on success, otherwise indicates + * the failure that occurred (other parameters are undefined if errorCode is nonzero). + * + * replyDomain: The name of the domain. + * + * context: The context pointer passed to DNSServiceEnumerateDomains. + * + */ + +typedef void (DNSSD_API *DNSServiceDomainEnumReply) + ( + DNSServiceRef sdRef, + DNSServiceFlags flags, + uint32_t interfaceIndex, + DNSServiceErrorType errorCode, + const char *replyDomain, + void *context + ); + + +/* DNSServiceEnumerateDomains() Parameters: + * + * + * sdRef: A pointer to an uninitialized DNSServiceRef. If the call succeeds + * then it initializes the DNSServiceRef, returns kDNSServiceErr_NoError, + * and the enumeration operation will run indefinitely until the client + * terminates it by passing this DNSServiceRef to DNSServiceRefDeallocate(). + * + * flags: Possible values are: + * kDNSServiceFlagsBrowseDomains to enumerate domains recommended for browsing. + * kDNSServiceFlagsRegistrationDomains to enumerate domains recommended + * for registration. + * + * interfaceIndex: If non-zero, specifies the interface on which to look for domains. + * (the index for a given interface is determined via the if_nametoindex() + * family of calls.) Most applications will pass 0 to enumerate domains on + * all interfaces. See "Constants for specifying an interface index" for more details. + * + * callBack: The function to be called when a domain is found or the call asynchronously + * fails. + * + * context: An application context pointer which is passed to the callback function + * (may be NULL). + * + * return value: Returns kDNSServiceErr_NoError on succeses (any subsequent, asynchronous + * errors are delivered to the callback), otherwise returns an error code indicating + * the error that occurred (the callback is not invoked and the DNSServiceRef + * is not initialized.) + */ + +DNSServiceErrorType DNSSD_API DNSServiceEnumerateDomains + ( + DNSServiceRef *sdRef, + DNSServiceFlags flags, + uint32_t interfaceIndex, + DNSServiceDomainEnumReply callBack, + void *context /* may be NULL */ + ); + + +/********************************************************************************************* + * + * Service Registration + * + *********************************************************************************************/ + +/* Register a service that is discovered via Browse() and Resolve() calls. + * + * + * DNSServiceRegisterReply() Callback Parameters: + * + * sdRef: The DNSServiceRef initialized by DNSServiceRegister(). + * + * flags: Currently unused, reserved for future use. + * + * errorCode: Will be kDNSServiceErr_NoError on success, otherwise will + * indicate the failure that occurred (including name conflicts, + * if the kDNSServiceFlagsNoAutoRename flag was used when registering.) + * Other parameters are undefined if errorCode is nonzero. + * + * name: The service name registered (if the application did not specify a name in + * DNSServiceRegister(), this indicates what name was automatically chosen). + * + * regtype: The type of service registered, as it was passed to the callout. + * + * domain: The domain on which the service was registered (if the application did not + * specify a domain in DNSServiceRegister(), this indicates the default domain + * on which the service was registered). + * + * context: The context pointer that was passed to the callout. + * + */ + +typedef void (DNSSD_API *DNSServiceRegisterReply) + ( + DNSServiceRef sdRef, + DNSServiceFlags flags, + DNSServiceErrorType errorCode, + const char *name, + const char *regtype, + const char *domain, + void *context + ); + + +/* DNSServiceRegister() Parameters: + * + * sdRef: A pointer to an uninitialized DNSServiceRef. If the call succeeds + * then it initializes the DNSServiceRef, returns kDNSServiceErr_NoError, + * and the registration will remain active indefinitely until the client + * terminates it by passing this DNSServiceRef to DNSServiceRefDeallocate(). + * + * interfaceIndex: If non-zero, specifies the interface on which to register the service + * (the index for a given interface is determined via the if_nametoindex() + * family of calls.) Most applications will pass 0 to register on all + * available interfaces. See "Constants for specifying an interface index" for more details. + * + * flags: Indicates the renaming behavior on name conflict (most applications + * will pass 0). See flag definitions above for details. + * + * name: If non-NULL, specifies the service name to be registered. + * Most applications will not specify a name, in which case the computer + * name is used (this name is communicated to the client via the callback). + * If a name is specified, it must be 1-63 bytes of UTF-8 text. + * If the name is longer than 63 bytes it will be automatically truncated + * to a legal length, unless the NoAutoRename flag is set, + * in which case kDNSServiceErr_BadParam will be returned. + * + * regtype: The service type followed by the protocol, separated by a dot + * (e.g. "_ftp._tcp"). The service type must be an underscore, followed + * by 1-14 characters, which may be letters, digits, or hyphens. + * The transport protocol must be "_tcp" or "_udp". New service types + * should be registered at . + * + * domain: If non-NULL, specifies the domain on which to advertise the service. + * Most applications will not specify a domain, instead automatically + * registering in the default domain(s). + * + * host: If non-NULL, specifies the SRV target host name. Most applications + * will not specify a host, instead automatically using the machine's + * default host name(s). Note that specifying a non-NULL host does NOT + * create an address record for that host - the application is responsible + * for ensuring that the appropriate address record exists, or creating it + * via DNSServiceRegisterRecord(). + * + * port: The port, in network byte order, on which the service accepts connections. + * Pass 0 for a "placeholder" service (i.e. a service that will not be discovered + * by browsing, but will cause a name conflict if another client tries to + * register that same name). Most clients will not use placeholder services. + * + * txtLen: The length of the txtRecord, in bytes. Must be zero if the txtRecord is NULL. + * + * txtRecord: The TXT record rdata. A non-NULL txtRecord MUST be a properly formatted DNS + * TXT record, i.e. ... + * Passing NULL for the txtRecord is allowed as a synonym for txtLen=1, txtRecord="", + * i.e. it creates a TXT record of length one containing a single empty string. + * RFC 1035 doesn't allow a TXT record to contain *zero* strings, so a single empty + * string is the smallest legal DNS TXT record. + * As with the other parameters, the DNSServiceRegister call copies the txtRecord + * data; e.g. if you allocated the storage for the txtRecord parameter with malloc() + * then you can safely free that memory right after the DNSServiceRegister call returns. + * + * callBack: The function to be called when the registration completes or asynchronously + * fails. The client MAY pass NULL for the callback - The client will NOT be notified + * of the default values picked on its behalf, and the client will NOT be notified of any + * asynchronous errors (e.g. out of memory errors, etc.) that may prevent the registration + * of the service. The client may NOT pass the NoAutoRename flag if the callback is NULL. + * The client may still deregister the service at any time via DNSServiceRefDeallocate(). + * + * context: An application context pointer which is passed to the callback function + * (may be NULL). + * + * return value: Returns kDNSServiceErr_NoError on succeses (any subsequent, asynchronous + * errors are delivered to the callback), otherwise returns an error code indicating + * the error that occurred (the callback is never invoked and the DNSServiceRef + * is not initialized.) + */ + +DNSServiceErrorType DNSSD_API DNSServiceRegister + ( + DNSServiceRef *sdRef, + DNSServiceFlags flags, + uint32_t interfaceIndex, + const char *name, /* may be NULL */ + const char *regtype, + const char *domain, /* may be NULL */ + const char *host, /* may be NULL */ + uint16_t port, + uint16_t txtLen, + const void *txtRecord, /* may be NULL */ + DNSServiceRegisterReply callBack, /* may be NULL */ + void *context /* may be NULL */ + ); + + +/* DNSServiceAddRecord() + * + * Add a record to a registered service. The name of the record will be the same as the + * registered service's name. + * The record can later be updated or deregistered by passing the RecordRef initialized + * by this function to DNSServiceUpdateRecord() or DNSServiceRemoveRecord(). + * + * Note that the DNSServiceAddRecord/UpdateRecord/RemoveRecord are *NOT* thread-safe + * with respect to a single DNSServiceRef. If you plan to have multiple threads + * in your program simultaneously add, update, or remove records from the same + * DNSServiceRef, then it's the caller's responsibility to use a mutext lock + * or take similar appropriate precautions to serialize those calls. + * + * + * Parameters; + * + * sdRef: A DNSServiceRef initialized by DNSServiceRegister(). + * + * RecordRef: A pointer to an uninitialized DNSRecordRef. Upon succesfull completion of this + * call, this ref may be passed to DNSServiceUpdateRecord() or DNSServiceRemoveRecord(). + * If the above DNSServiceRef is passed to DNSServiceRefDeallocate(), RecordRef is also + * invalidated and may not be used further. + * + * flags: Currently ignored, reserved for future use. + * + * rrtype: The type of the record (e.g. kDNSServiceType_TXT, kDNSServiceType_SRV, etc) + * + * rdlen: The length, in bytes, of the rdata. + * + * rdata: The raw rdata to be contained in the added resource record. + * + * ttl: The time to live of the resource record, in seconds. Pass 0 to use a default value. + * + * return value: Returns kDNSServiceErr_NoError on success, otherwise returns an + * error code indicating the error that occurred (the RecordRef is not initialized). + */ + +DNSServiceErrorType DNSSD_API DNSServiceAddRecord + ( + DNSServiceRef sdRef, + DNSRecordRef *RecordRef, + DNSServiceFlags flags, + uint16_t rrtype, + uint16_t rdlen, + const void *rdata, + uint32_t ttl + ); + + +/* DNSServiceUpdateRecord + * + * Update a registered resource record. The record must either be: + * - The primary txt record of a service registered via DNSServiceRegister() + * - A record added to a registered service via DNSServiceAddRecord() + * - An individual record registered by DNSServiceRegisterRecord() + * + * + * Parameters: + * + * sdRef: A DNSServiceRef that was initialized by DNSServiceRegister() + * or DNSServiceCreateConnection(). + * + * RecordRef: A DNSRecordRef initialized by DNSServiceAddRecord, or NULL to update the + * service's primary txt record. + * + * flags: Currently ignored, reserved for future use. + * + * rdlen: The length, in bytes, of the new rdata. + * + * rdata: The new rdata to be contained in the updated resource record. + * + * ttl: The time to live of the updated resource record, in seconds. + * + * return value: Returns kDNSServiceErr_NoError on success, otherwise returns an + * error code indicating the error that occurred. + */ + +DNSServiceErrorType DNSSD_API DNSServiceUpdateRecord + ( + DNSServiceRef sdRef, + DNSRecordRef RecordRef, /* may be NULL */ + DNSServiceFlags flags, + uint16_t rdlen, + const void *rdata, + uint32_t ttl + ); + + +/* DNSServiceRemoveRecord + * + * Remove a record previously added to a service record set via DNSServiceAddRecord(), or deregister + * an record registered individually via DNSServiceRegisterRecord(). + * + * Parameters: + * + * sdRef: A DNSServiceRef initialized by DNSServiceRegister() (if the + * record being removed was registered via DNSServiceAddRecord()) or by + * DNSServiceCreateConnection() (if the record being removed was registered via + * DNSServiceRegisterRecord()). + * + * recordRef: A DNSRecordRef initialized by a successful call to DNSServiceAddRecord() + * or DNSServiceRegisterRecord(). + * + * flags: Currently ignored, reserved for future use. + * + * return value: Returns kDNSServiceErr_NoError on success, otherwise returns an + * error code indicating the error that occurred. + */ + +DNSServiceErrorType DNSSD_API DNSServiceRemoveRecord + ( + DNSServiceRef sdRef, + DNSRecordRef RecordRef, + DNSServiceFlags flags + ); + + +/********************************************************************************************* + * + * Service Discovery + * + *********************************************************************************************/ + +/* Browse for instances of a service. + * + * + * DNSServiceBrowseReply() Parameters: + * + * sdRef: The DNSServiceRef initialized by DNSServiceBrowse(). + * + * flags: Possible values are kDNSServiceFlagsMoreComing and kDNSServiceFlagsAdd. + * See flag definitions for details. + * + * interfaceIndex: The interface on which the service is advertised. This index should + * be passed to DNSServiceResolve() when resolving the service. + * + * errorCode: Will be kDNSServiceErr_NoError (0) on success, otherwise will + * indicate the failure that occurred. Other parameters are undefined if + * the errorCode is nonzero. + * + * serviceName: The discovered service name. This name should be displayed to the user, + * and stored for subsequent use in the DNSServiceResolve() call. + * + * regtype: The service type, which is usually (but not always) the same as was passed + * to DNSServiceBrowse(). One case where the discovered service type may + * not be the same as the requested service type is when using subtypes: + * The client may want to browse for only those ftp servers that allow + * anonymous connections. The client will pass the string "_ftp._tcp,_anon" + * to DNSServiceBrowse(), but the type of the service that's discovered + * is simply "_ftp._tcp". The regtype for each discovered service instance + * should be stored along with the name, so that it can be passed to + * DNSServiceResolve() when the service is later resolved. + * + * domain: The domain of the discovered service instance. This may or may not be the + * same as the domain that was passed to DNSServiceBrowse(). The domain for each + * discovered service instance should be stored along with the name, so that + * it can be passed to DNSServiceResolve() when the service is later resolved. + * + * context: The context pointer that was passed to the callout. + * + */ + +typedef void (DNSSD_API *DNSServiceBrowseReply) + ( + DNSServiceRef sdRef, + DNSServiceFlags flags, + uint32_t interfaceIndex, + DNSServiceErrorType errorCode, + const char *serviceName, + const char *regtype, + const char *replyDomain, + void *context + ); + + +/* DNSServiceBrowse() Parameters: + * + * sdRef: A pointer to an uninitialized DNSServiceRef. If the call succeeds + * then it initializes the DNSServiceRef, returns kDNSServiceErr_NoError, + * and the browse operation will run indefinitely until the client + * terminates it by passing this DNSServiceRef to DNSServiceRefDeallocate(). + * + * flags: Currently ignored, reserved for future use. + * + * interfaceIndex: If non-zero, specifies the interface on which to browse for services + * (the index for a given interface is determined via the if_nametoindex() + * family of calls.) Most applications will pass 0 to browse on all available + * interfaces. See "Constants for specifying an interface index" for more details. + * + * regtype: The service type being browsed for followed by the protocol, separated by a + * dot (e.g. "_ftp._tcp"). The transport protocol must be "_tcp" or "_udp". + * + * domain: If non-NULL, specifies the domain on which to browse for services. + * Most applications will not specify a domain, instead browsing on the + * default domain(s). + * + * callBack: The function to be called when an instance of the service being browsed for + * is found, or if the call asynchronously fails. + * + * context: An application context pointer which is passed to the callback function + * (may be NULL). + * + * return value: Returns kDNSServiceErr_NoError on succeses (any subsequent, asynchronous + * errors are delivered to the callback), otherwise returns an error code indicating + * the error that occurred (the callback is not invoked and the DNSServiceRef + * is not initialized.) + */ + +DNSServiceErrorType DNSSD_API DNSServiceBrowse + ( + DNSServiceRef *sdRef, + DNSServiceFlags flags, + uint32_t interfaceIndex, + const char *regtype, + const char *domain, /* may be NULL */ + DNSServiceBrowseReply callBack, + void *context /* may be NULL */ + ); + + +/* DNSServiceResolve() + * + * Resolve a service name discovered via DNSServiceBrowse() to a target host name, port number, and + * txt record. + * + * Note: Applications should NOT use DNSServiceResolve() solely for txt record monitoring - use + * DNSServiceQueryRecord() instead, as it is more efficient for this task. + * + * Note: When the desired results have been returned, the client MUST terminate the resolve by calling + * DNSServiceRefDeallocate(). + * + * Note: DNSServiceResolve() behaves correctly for typical services that have a single SRV record + * and a single TXT record. To resolve non-standard services with multiple SRV or TXT records, + * DNSServiceQueryRecord() should be used. + * + * DNSServiceResolveReply Callback Parameters: + * + * sdRef: The DNSServiceRef initialized by DNSServiceResolve(). + * + * flags: Currently unused, reserved for future use. + * + * interfaceIndex: The interface on which the service was resolved. + * + * errorCode: Will be kDNSServiceErr_NoError (0) on success, otherwise will + * indicate the failure that occurred. Other parameters are undefined if + * the errorCode is nonzero. + * + * fullname: The full service domain name, in the form ... + * (This name is escaped following standard DNS rules, making it suitable for + * passing to standard system DNS APIs such as res_query(), or to the + * special-purpose functions included in this API that take fullname parameters. + * See "Notes on DNS Name Escaping" earlier in this file for more details.) + * + * hosttarget: The target hostname of the machine providing the service. This name can + * be passed to functions like gethostbyname() to identify the host's IP address. + * + * port: The port, in network byte order, on which connections are accepted for this service. + * + * txtLen: The length of the txt record, in bytes. + * + * txtRecord: The service's primary txt record, in standard txt record format. + * + * context: The context pointer that was passed to the callout. + * + * NOTE: In earlier versions of this header file, the txtRecord parameter was declared "const char *" + * This is incorrect, since it contains length bytes which are values in the range 0 to 255, not -128 to +127. + * Depending on your compiler settings, this change may cause signed/unsigned mismatch warnings. + * These should be fixed by updating your own callback function definition to match the corrected + * function signature using "const unsigned char *txtRecord". Making this change may also fix inadvertent + * bugs in your callback function, where it could have incorrectly interpreted a length byte with value 250 + * as being -6 instead, with various bad consequences ranging from incorrect operation to software crashes. + * If you need to maintain portable code that will compile cleanly with both the old and new versions of + * this header file, you should update your callback function definition to use the correct unsigned value, + * and then in the place where you pass your callback function to DNSServiceResolve(), use a cast to eliminate + * the compiler warning, e.g.: + * DNSServiceResolve(sd, flags, index, name, regtype, domain, (DNSServiceResolveReply)MyCallback, context); + * This will ensure that your code compiles cleanly without warnings (and more importantly, works correctly) + * with both the old header and with the new corrected version. + * + */ + +typedef void (DNSSD_API *DNSServiceResolveReply) + ( + DNSServiceRef sdRef, + DNSServiceFlags flags, + uint32_t interfaceIndex, + DNSServiceErrorType errorCode, + const char *fullname, + const char *hosttarget, + uint16_t port, + uint16_t txtLen, + const unsigned char *txtRecord, + void *context + ); + + +/* DNSServiceResolve() Parameters + * + * sdRef: A pointer to an uninitialized DNSServiceRef. If the call succeeds + * then it initializes the DNSServiceRef, returns kDNSServiceErr_NoError, + * and the resolve operation will run indefinitely until the client + * terminates it by passing this DNSServiceRef to DNSServiceRefDeallocate(). + * + * flags: Currently ignored, reserved for future use. + * + * interfaceIndex: The interface on which to resolve the service. If this resolve call is + * as a result of a currently active DNSServiceBrowse() operation, then the + * interfaceIndex should be the index reported in the DNSServiceBrowseReply + * callback. If this resolve call is using information previously saved + * (e.g. in a preference file) for later use, then use interfaceIndex 0, because + * the desired service may now be reachable via a different physical interface. + * See "Constants for specifying an interface index" for more details. + * + * name: The name of the service instance to be resolved, as reported to the + * DNSServiceBrowseReply() callback. + * + * regtype: The type of the service instance to be resolved, as reported to the + * DNSServiceBrowseReply() callback. + * + * domain: The domain of the service instance to be resolved, as reported to the + * DNSServiceBrowseReply() callback. + * + * callBack: The function to be called when a result is found, or if the call + * asynchronously fails. + * + * context: An application context pointer which is passed to the callback function + * (may be NULL). + * + * return value: Returns kDNSServiceErr_NoError on succeses (any subsequent, asynchronous + * errors are delivered to the callback), otherwise returns an error code indicating + * the error that occurred (the callback is never invoked and the DNSServiceRef + * is not initialized.) + */ + +DNSServiceErrorType DNSSD_API DNSServiceResolve + ( + DNSServiceRef *sdRef, + DNSServiceFlags flags, + uint32_t interfaceIndex, + const char *name, + const char *regtype, + const char *domain, + DNSServiceResolveReply callBack, + void *context /* may be NULL */ + ); + + +/********************************************************************************************* + * + * Special Purpose Calls (most applications will not use these) + * + *********************************************************************************************/ + +/* DNSServiceCreateConnection() + * + * Create a connection to the daemon allowing efficient registration of + * multiple individual records. + * + * + * Parameters: + * + * sdRef: A pointer to an uninitialized DNSServiceRef. Deallocating + * the reference (via DNSServiceRefDeallocate()) severs the + * connection and deregisters all records registered on this connection. + * + * return value: Returns kDNSServiceErr_NoError on success, otherwise returns + * an error code indicating the specific failure that occurred (in which + * case the DNSServiceRef is not initialized). + */ + +DNSServiceErrorType DNSSD_API DNSServiceCreateConnection(DNSServiceRef *sdRef); + + +/* DNSServiceRegisterRecord + * + * Register an individual resource record on a connected DNSServiceRef. + * + * Note that name conflicts occurring for records registered via this call must be handled + * by the client in the callback. + * + * + * DNSServiceRegisterRecordReply() parameters: + * + * sdRef: The connected DNSServiceRef initialized by + * DNSServiceCreateConnection(). + * + * RecordRef: The DNSRecordRef initialized by DNSServiceRegisterRecord(). If the above + * DNSServiceRef is passed to DNSServiceRefDeallocate(), this DNSRecordRef is + * invalidated, and may not be used further. + * + * flags: Currently unused, reserved for future use. + * + * errorCode: Will be kDNSServiceErr_NoError on success, otherwise will + * indicate the failure that occurred (including name conflicts.) + * Other parameters are undefined if errorCode is nonzero. + * + * context: The context pointer that was passed to the callout. + * + */ + + typedef void (DNSSD_API *DNSServiceRegisterRecordReply) + ( + DNSServiceRef sdRef, + DNSRecordRef RecordRef, + DNSServiceFlags flags, + DNSServiceErrorType errorCode, + void *context + ); + + +/* DNSServiceRegisterRecord() Parameters: + * + * sdRef: A DNSServiceRef initialized by DNSServiceCreateConnection(). + * + * RecordRef: A pointer to an uninitialized DNSRecordRef. Upon succesfull completion of this + * call, this ref may be passed to DNSServiceUpdateRecord() or DNSServiceRemoveRecord(). + * (To deregister ALL records registered on a single connected DNSServiceRef + * and deallocate each of their corresponding DNSServiceRecordRefs, call + * DNSServiceRefDealloocate()). + * + * flags: Possible values are kDNSServiceFlagsShared or kDNSServiceFlagsUnique + * (see flag type definitions for details). + * + * interfaceIndex: If non-zero, specifies the interface on which to register the record + * (the index for a given interface is determined via the if_nametoindex() + * family of calls.) Passing 0 causes the record to be registered on all interfaces. + * See "Constants for specifying an interface index" for more details. + * + * fullname: The full domain name of the resource record. + * + * rrtype: The numerical type of the resource record (e.g. kDNSServiceType_PTR, kDNSServiceType_SRV, etc) + * + * rrclass: The class of the resource record (usually kDNSServiceClass_IN) + * + * rdlen: Length, in bytes, of the rdata. + * + * rdata: A pointer to the raw rdata, as it is to appear in the DNS record. + * + * ttl: The time to live of the resource record, in seconds. Pass 0 to use a default value. + * + * callBack: The function to be called when a result is found, or if the call + * asynchronously fails (e.g. because of a name conflict.) + * + * context: An application context pointer which is passed to the callback function + * (may be NULL). + * + * return value: Returns kDNSServiceErr_NoError on succeses (any subsequent, asynchronous + * errors are delivered to the callback), otherwise returns an error code indicating + * the error that occurred (the callback is never invoked and the DNSRecordRef is + * not initialized.) + */ + +DNSServiceErrorType DNSSD_API DNSServiceRegisterRecord + ( + DNSServiceRef sdRef, + DNSRecordRef *RecordRef, + DNSServiceFlags flags, + uint32_t interfaceIndex, + const char *fullname, + uint16_t rrtype, + uint16_t rrclass, + uint16_t rdlen, + const void *rdata, + uint32_t ttl, + DNSServiceRegisterRecordReply callBack, + void *context /* may be NULL */ + ); + + +/* DNSServiceQueryRecord + * + * Query for an arbitrary DNS record. + * + * + * DNSServiceQueryRecordReply() Callback Parameters: + * + * sdRef: The DNSServiceRef initialized by DNSServiceQueryRecord(). + * + * flags: Possible values are kDNSServiceFlagsMoreComing and + * kDNSServiceFlagsAdd. The Add flag is NOT set for PTR records + * with a ttl of 0, i.e. "Remove" events. + * + * interfaceIndex: The interface on which the query was resolved (the index for a given + * interface is determined via the if_nametoindex() family of calls). + * See "Constants for specifying an interface index" for more details. + * + * errorCode: Will be kDNSServiceErr_NoError on success, otherwise will + * indicate the failure that occurred. Other parameters are undefined if + * errorCode is nonzero. + * + * fullname: The resource record's full domain name. + * + * rrtype: The resource record's type (e.g. kDNSServiceType_PTR, kDNSServiceType_SRV, etc) + * + * rrclass: The class of the resource record (usually kDNSServiceClass_IN). + * + * rdlen: The length, in bytes, of the resource record rdata. + * + * rdata: The raw rdata of the resource record. + * + * ttl: The resource record's time to live, in seconds. + * + * context: The context pointer that was passed to the callout. + * + */ + +typedef void (DNSSD_API *DNSServiceQueryRecordReply) + ( + DNSServiceRef DNSServiceRef, + DNSServiceFlags flags, + uint32_t interfaceIndex, + DNSServiceErrorType errorCode, + const char *fullname, + uint16_t rrtype, + uint16_t rrclass, + uint16_t rdlen, + const void *rdata, + uint32_t ttl, + void *context + ); + + +/* DNSServiceQueryRecord() Parameters: + * + * sdRef: A pointer to an uninitialized DNSServiceRef. If the call succeeds + * then it initializes the DNSServiceRef, returns kDNSServiceErr_NoError, + * and the query operation will run indefinitely until the client + * terminates it by passing this DNSServiceRef to DNSServiceRefDeallocate(). + * + * flags: Pass kDNSServiceFlagsLongLivedQuery to create a "long-lived" unicast + * query in a non-local domain. Without setting this flag, unicast queries + * will be one-shot - that is, only answers available at the time of the call + * will be returned. By setting this flag, answers (including Add and Remove + * events) that become available after the initial call is made will generate + * callbacks. This flag has no effect on link-local multicast queries. + * + * interfaceIndex: If non-zero, specifies the interface on which to issue the query + * (the index for a given interface is determined via the if_nametoindex() + * family of calls.) Passing 0 causes the name to be queried for on all + * interfaces. See "Constants for specifying an interface index" for more details. + * + * fullname: The full domain name of the resource record to be queried for. + * + * rrtype: The numerical type of the resource record to be queried for + * (e.g. kDNSServiceType_PTR, kDNSServiceType_SRV, etc) + * + * rrclass: The class of the resource record (usually kDNSServiceClass_IN). + * + * callBack: The function to be called when a result is found, or if the call + * asynchronously fails. + * + * context: An application context pointer which is passed to the callback function + * (may be NULL). + * + * return value: Returns kDNSServiceErr_NoError on succeses (any subsequent, asynchronous + * errors are delivered to the callback), otherwise returns an error code indicating + * the error that occurred (the callback is never invoked and the DNSServiceRef + * is not initialized.) + */ + +DNSServiceErrorType DNSSD_API DNSServiceQueryRecord + ( + DNSServiceRef *sdRef, + DNSServiceFlags flags, + uint32_t interfaceIndex, + const char *fullname, + uint16_t rrtype, + uint16_t rrclass, + DNSServiceQueryRecordReply callBack, + void *context /* may be NULL */ + ); + + +/* DNSServiceReconfirmRecord + * + * Instruct the daemon to verify the validity of a resource record that appears to + * be out of date (e.g. because tcp connection to a service's target failed.) + * Causes the record to be flushed from the daemon's cache (as well as all other + * daemons' caches on the network) if the record is determined to be invalid. + * + * Parameters: + * + * flags: Currently unused, reserved for future use. + * + * interfaceIndex: If non-zero, specifies the interface of the record in question. + * Passing 0 causes all instances of this record to be reconfirmed. + * + * fullname: The resource record's full domain name. + * + * rrtype: The resource record's type (e.g. kDNSServiceType_PTR, kDNSServiceType_SRV, etc) + * + * rrclass: The class of the resource record (usually kDNSServiceClass_IN). + * + * rdlen: The length, in bytes, of the resource record rdata. + * + * rdata: The raw rdata of the resource record. + * + */ + +DNSServiceErrorType DNSSD_API DNSServiceReconfirmRecord + ( + DNSServiceFlags flags, + uint32_t interfaceIndex, + const char *fullname, + uint16_t rrtype, + uint16_t rrclass, + uint16_t rdlen, + const void *rdata + ); + + +/********************************************************************************************* + * + * General Utility Functions + * + *********************************************************************************************/ + +/* DNSServiceConstructFullName() + * + * Concatenate a three-part domain name (as returned by the above callbacks) into a + * properly-escaped full domain name. Note that callbacks in the above functions ALREADY ESCAPE + * strings where necessary. + * + * Parameters: + * + * fullName: A pointer to a buffer that where the resulting full domain name is to be written. + * The buffer must be kDNSServiceMaxDomainName (1005) bytes in length to + * accommodate the longest legal domain name without buffer overrun. + * + * service: The service name - any dots or backslashes must NOT be escaped. + * May be NULL (to construct a PTR record name, e.g. + * "_ftp._tcp.apple.com."). + * + * regtype: The service type followed by the protocol, separated by a dot + * (e.g. "_ftp._tcp"). + * + * domain: The domain name, e.g. "apple.com.". Literal dots or backslashes, + * if any, must be escaped, e.g. "1st\. Floor.apple.com." + * + * return value: Returns 0 on success, -1 on error. + * + */ + +int DNSSD_API DNSServiceConstructFullName + ( + char *fullName, + const char *service, /* may be NULL */ + const char *regtype, + const char *domain + ); + + +/********************************************************************************************* + * + * TXT Record Construction Functions + * + *********************************************************************************************/ + +/* + * A typical calling sequence for TXT record construction is something like: + * + * Client allocates storage for TXTRecord data (e.g. declare buffer on the stack) + * TXTRecordCreate(); + * TXTRecordSetValue(); + * TXTRecordSetValue(); + * TXTRecordSetValue(); + * ... + * DNSServiceRegister( ... TXTRecordGetLength(), TXTRecordGetBytesPtr() ... ); + * TXTRecordDeallocate(); + * Explicitly deallocate storage for TXTRecord data (if not allocated on the stack) + */ + + +/* TXTRecordRef + * + * Opaque internal data type. + * Note: Represents a DNS-SD TXT record. + */ + +typedef union _TXTRecordRef_t { char PrivateData[16]; char *ForceNaturalAlignment; } TXTRecordRef; + + +/* TXTRecordCreate() + * + * Creates a new empty TXTRecordRef referencing the specified storage. + * + * If the buffer parameter is NULL, or the specified storage size is not + * large enough to hold a key subsequently added using TXTRecordSetValue(), + * then additional memory will be added as needed using malloc(). + * + * On some platforms, when memory is low, malloc() may fail. In this + * case, TXTRecordSetValue() will return kDNSServiceErr_NoMemory, and this + * error condition will need to be handled as appropriate by the caller. + * + * You can avoid the need to handle this error condition if you ensure + * that the storage you initially provide is large enough to hold all + * the key/value pairs that are to be added to the record. + * The caller can precompute the exact length required for all of the + * key/value pairs to be added, or simply provide a fixed-sized buffer + * known in advance to be large enough. + * A no-value (key-only) key requires (1 + key length) bytes. + * A key with empty value requires (1 + key length + 1) bytes. + * A key with non-empty value requires (1 + key length + 1 + value length). + * For most applications, DNS-SD TXT records are generally + * less than 100 bytes, so in most cases a simple fixed-sized + * 256-byte buffer will be more than sufficient. + * Recommended size limits for DNS-SD TXT Records are discussed in + * + * + * Note: When passing parameters to and from these TXT record APIs, + * the key name does not include the '=' character. The '=' character + * is the separator between the key and value in the on-the-wire + * packet format; it is not part of either the key or the value. + * + * txtRecord: A pointer to an uninitialized TXTRecordRef. + * + * bufferLen: The size of the storage provided in the "buffer" parameter. + * + * buffer: Optional caller-supplied storage used to hold the TXTRecord data. + * This storage must remain valid for as long as + * the TXTRecordRef. + */ + +void DNSSD_API TXTRecordCreate + ( + TXTRecordRef *txtRecord, + uint16_t bufferLen, + void *buffer + ); + + +/* TXTRecordDeallocate() + * + * Releases any resources allocated in the course of preparing a TXT Record + * using TXTRecordCreate()/TXTRecordSetValue()/TXTRecordRemoveValue(). + * Ownership of the buffer provided in TXTRecordCreate() returns to the client. + * + * txtRecord: A TXTRecordRef initialized by calling TXTRecordCreate(). + * + */ + +void DNSSD_API TXTRecordDeallocate + ( + TXTRecordRef *txtRecord + ); + + +/* TXTRecordSetValue() + * + * Adds a key (optionally with value) to a TXTRecordRef. If the "key" already + * exists in the TXTRecordRef, then the current value will be replaced with + * the new value. + * Keys may exist in four states with respect to a given TXT record: + * - Absent (key does not appear at all) + * - Present with no value ("key" appears alone) + * - Present with empty value ("key=" appears in TXT record) + * - Present with non-empty value ("key=value" appears in TXT record) + * For more details refer to "Data Syntax for DNS-SD TXT Records" in + * + * + * txtRecord: A TXTRecordRef initialized by calling TXTRecordCreate(). + * + * key: A null-terminated string which only contains printable ASCII + * values (0x20-0x7E), excluding '=' (0x3D). Keys should be + * 8 characters or less (not counting the terminating null). + * + * valueSize: The size of the value. + * + * value: Any binary value. For values that represent + * textual data, UTF-8 is STRONGLY recommended. + * For values that represent textual data, valueSize + * should NOT include the terminating null (if any) + * at the end of the string. + * If NULL, then "key" will be added with no value. + * If non-NULL but valueSize is zero, then "key=" will be + * added with empty value. + * + * return value: Returns kDNSServiceErr_NoError on success. + * Returns kDNSServiceErr_Invalid if the "key" string contains + * illegal characters. + * Returns kDNSServiceErr_NoMemory if adding this key would + * exceed the available storage. + */ + +DNSServiceErrorType DNSSD_API TXTRecordSetValue + ( + TXTRecordRef *txtRecord, + const char *key, + uint8_t valueSize, /* may be zero */ + const void *value /* may be NULL */ + ); + + +/* TXTRecordRemoveValue() + * + * Removes a key from a TXTRecordRef. The "key" must be an + * ASCII string which exists in the TXTRecordRef. + * + * txtRecord: A TXTRecordRef initialized by calling TXTRecordCreate(). + * + * key: A key name which exists in the TXTRecordRef. + * + * return value: Returns kDNSServiceErr_NoError on success. + * Returns kDNSServiceErr_NoSuchKey if the "key" does not + * exist in the TXTRecordRef. + */ + +DNSServiceErrorType DNSSD_API TXTRecordRemoveValue + ( + TXTRecordRef *txtRecord, + const char *key + ); + + +/* TXTRecordGetLength() + * + * Allows you to determine the length of the raw bytes within a TXTRecordRef. + * + * txtRecord: A TXTRecordRef initialized by calling TXTRecordCreate(). + * + * return value: Returns the size of the raw bytes inside a TXTRecordRef + * which you can pass directly to DNSServiceRegister() or + * to DNSServiceUpdateRecord(). + * Returns 0 if the TXTRecordRef is empty. + */ + +uint16_t DNSSD_API TXTRecordGetLength + ( + const TXTRecordRef *txtRecord + ); + + +/* TXTRecordGetBytesPtr() + * + * Allows you to retrieve a pointer to the raw bytes within a TXTRecordRef. + * + * txtRecord: A TXTRecordRef initialized by calling TXTRecordCreate(). + * + * return value: Returns a pointer to the raw bytes inside the TXTRecordRef + * which you can pass directly to DNSServiceRegister() or + * to DNSServiceUpdateRecord(). + */ + +const void * DNSSD_API TXTRecordGetBytesPtr + ( + const TXTRecordRef *txtRecord + ); + + +/********************************************************************************************* + * + * TXT Record Parsing Functions + * + *********************************************************************************************/ + +/* + * A typical calling sequence for TXT record parsing is something like: + * + * Receive TXT record data in DNSServiceResolve() callback + * if (TXTRecordContainsKey(txtLen, txtRecord, "key")) then do something + * val1ptr = TXTRecordGetValuePtr(txtLen, txtRecord, "key1", &len1); + * val2ptr = TXTRecordGetValuePtr(txtLen, txtRecord, "key2", &len2); + * ... + * bcopy(val1ptr, myval1, len1); + * bcopy(val2ptr, myval2, len2); + * ... + * return; + * + * If you wish to retain the values after return from the DNSServiceResolve() + * callback, then you need to copy the data to your own storage using bcopy() + * or similar, as shown in the example above. + * + * If for some reason you need to parse a TXT record you built yourself + * using the TXT record construction functions above, then you can do + * that using TXTRecordGetLength and TXTRecordGetBytesPtr calls: + * TXTRecordGetValue(TXTRecordGetLength(x), TXTRecordGetBytesPtr(x), key, &len); + * + * Most applications only fetch keys they know about from a TXT record and + * ignore the rest. + * However, some debugging tools wish to fetch and display all keys. + * To do that, use the TXTRecordGetCount() and TXTRecordGetItemAtIndex() calls. + */ + +/* TXTRecordContainsKey() + * + * Allows you to determine if a given TXT Record contains a specified key. + * + * txtLen: The size of the received TXT Record. + * + * txtRecord: Pointer to the received TXT Record bytes. + * + * key: A null-terminated ASCII string containing the key name. + * + * return value: Returns 1 if the TXT Record contains the specified key. + * Otherwise, it returns 0. + */ + +int DNSSD_API TXTRecordContainsKey + ( + uint16_t txtLen, + const void *txtRecord, + const char *key + ); + + +/* TXTRecordGetValuePtr() + * + * Allows you to retrieve the value for a given key from a TXT Record. + * + * txtLen: The size of the received TXT Record + * + * txtRecord: Pointer to the received TXT Record bytes. + * + * key: A null-terminated ASCII string containing the key name. + * + * valueLen: On output, will be set to the size of the "value" data. + * + * return value: Returns NULL if the key does not exist in this TXT record, + * or exists with no value (to differentiate between + * these two cases use TXTRecordContainsKey()). + * Returns pointer to location within TXT Record bytes + * if the key exists with empty or non-empty value. + * For empty value, valueLen will be zero. + * For non-empty value, valueLen will be length of value data. + */ + +const void * DNSSD_API TXTRecordGetValuePtr + ( + uint16_t txtLen, + const void *txtRecord, + const char *key, + uint8_t *valueLen + ); + + +/* TXTRecordGetCount() + * + * Returns the number of keys stored in the TXT Record. The count + * can be used with TXTRecordGetItemAtIndex() to iterate through the keys. + * + * txtLen: The size of the received TXT Record. + * + * txtRecord: Pointer to the received TXT Record bytes. + * + * return value: Returns the total number of keys in the TXT Record. + * + */ + +uint16_t DNSSD_API TXTRecordGetCount + ( + uint16_t txtLen, + const void *txtRecord + ); + + +/* TXTRecordGetItemAtIndex() + * + * Allows you to retrieve a key name and value pointer, given an index into + * a TXT Record. Legal index values range from zero to TXTRecordGetCount()-1. + * It's also possible to iterate through keys in a TXT record by simply + * calling TXTRecordGetItemAtIndex() repeatedly, beginning with index zero + * and increasing until TXTRecordGetItemAtIndex() returns kDNSServiceErr_Invalid. + * + * On return: + * For keys with no value, *value is set to NULL and *valueLen is zero. + * For keys with empty value, *value is non-NULL and *valueLen is zero. + * For keys with non-empty value, *value is non-NULL and *valueLen is non-zero. + * + * txtLen: The size of the received TXT Record. + * + * txtRecord: Pointer to the received TXT Record bytes. + * + * index: An index into the TXT Record. + * + * keyBufLen: The size of the string buffer being supplied. + * + * key: A string buffer used to store the key name. + * On return, the buffer contains a null-terminated C string + * giving the key name. DNS-SD TXT keys are usually + * 8 characters or less. To hold the maximum possible + * key name, the buffer should be 256 bytes long. + * + * valueLen: On output, will be set to the size of the "value" data. + * + * value: On output, *value is set to point to location within TXT + * Record bytes that holds the value data. + * + * return value: Returns kDNSServiceErr_NoError on success. + * Returns kDNSServiceErr_NoMemory if keyBufLen is too short. + * Returns kDNSServiceErr_Invalid if index is greater than + * TXTRecordGetCount()-1. + */ + +DNSServiceErrorType DNSSD_API TXTRecordGetItemAtIndex + ( + uint16_t txtLen, + const void *txtRecord, + uint16_t index, + uint16_t keyBufLen, + char *key, + uint8_t *valueLen, + const void **value + ); + +#ifdef __APPLE_API_PRIVATE + +/* + * Mac OS X specific functionality + * 3rd party clients of this API should not depend on future support or availability of this routine + */ + +/* DNSServiceSetDefaultDomainForUser() + * + * Set the default domain for the caller's UID. Future browse and registration + * calls by this user that do not specify an explicit domain will browse and + * register in this wide-area domain in addition to .local. In addition, this + * domain will be returned as a Browse domain via domain enumeration calls. + * + * + * Parameters: + * + * flags: Pass kDNSServiceFlagsAdd to add a domain for a user. Call without + * this flag set to clear a previously added domain. + * + * domain: The domain to be used for the caller's UID. + * + * return value: Returns kDNSServiceErr_NoError on succeses, otherwise returns + * an error code indicating the error that occurred + */ + +DNSServiceErrorType DNSSD_API DNSServiceSetDefaultDomainForUser + ( + DNSServiceFlags flags, + const char *domain + ); + +#endif //__APPLE_API_PRIVATE + +// Some C compiler cleverness. We can make the compiler check certain things for us, +// and report errors at compile-time if anything is wrong. The usual way to do this would +// be to use a run-time "if" statement or the conventional run-time "assert" mechanism, but +// then you don't find out what's wrong until you run the software. This way, if the assertion +// condition is false, the array size is negative, and the complier complains immediately. + +struct DNS_SD_CompileTimeAssertionChecks + { + char assert0[(sizeof(union _TXTRecordRef_t) == 16) ? 1 : -1]; + }; + +#ifdef __cplusplus + } +#endif + +#endif /* _DNS_SD_H */ diff --git a/pkg/vere/io/ames.c b/pkg/vere/io/ames.c index 24baa6e042..6dd6cff67d 100644 --- a/pkg/vere/io/ames.c +++ b/pkg/vere/io/ames.c @@ -2060,6 +2060,50 @@ _ames_recv_cb(uv_udp_t* wax_u, } } +static void +_mdns_dear_bail(u3_ovum* egg_u, u3_noun lud) +{ + u3l_log("mdns: %%dear failure;"); + u3z(lud); + u3_ovum_free(egg_u); +} + +/* _ames_put_dear(): send lane to arvo after hearing mdns response +*/ +static void +_ames_put_dear(char* ship, unsigned long s_addr, uint16_t port, void* context) +{ + u3_ames* sam_u = (u3_ames*)context; + + u3_lane lan; + lan.pip_w = ntohl(s_addr); + lan.por_s = ntohs(port); + + u3_noun shp_u = u3dc("slaw", c3__p, u3i_string(ship)); + + if (u3_nul == shp_u) { + u3l_log("ames: strange ship from mdns: %s", ship); + return; + } + + u3_noun our = u3i_chubs(2, sam_u->pir_u->who_d); + if (our == u3t(shp_u)) { + u3z(our); + return; + } + + u3z(our); + + u3_noun wir = u3nc(c3__ames, u3_nul); + u3_noun cad = u3nt(c3__dear, u3k(u3t(shp_u)), u3nc(c3n, u3_ames_encode_lane(lan))); + + u3_auto_peer( + u3_auto_plan(&sam_u->car_u, + u3_ovum_init(0, c3__a, wir, cad)), + 0, 0, _mdns_dear_bail); + u3z(shp_u); +} + /* _ames_io_start(): initialize ames I/O. */ static void @@ -2124,6 +2168,12 @@ _ames_io_start(u3_ames* sam_u) u3l_log("ames: live on %d (localhost only)", sam_u->pir_u->por_s); } + u3_noun our = u3dc("scot", 'p', who); + char* our_s = u3r_string(our); + u3z(our); + + mdns_init(por_s, our_s, _ames_put_dear, (void *)sam_u); + uv_udp_recv_start(&sam_u->wax_u, _ames_alloc, _ames_recv_cb); sam_u->car_u.liv_o = c3y; diff --git a/pkg/vere/mdns.c b/pkg/vere/mdns.c new file mode 100644 index 0000000000..54fd797cf4 --- /dev/null +++ b/pkg/vere/mdns.c @@ -0,0 +1,226 @@ +/// @file + +#include "vere.h" + +#include "dns_sd.h" + +typedef struct _mdns_payload { + mdns_cb* cb; + DNSServiceRef sref; + char* who; + char* our; + uint16_t port; + uv_poll_t* poll; + void* context; +} mdns_payload; + + +static void getaddrinfo_cb(uv_getaddrinfo_t* req, int status, struct addrinfo* res) { + mdns_payload* payload = (mdns_payload*)req->data; + + if (status < 0) { + u3l_log("mdns: getaddrinfo error: %s", uv_strerror(status)); + uv_poll_stop(payload->poll); + c3_free(payload->poll); + DNSServiceRefDeallocate(payload->sref); + c3_free(payload); + c3_free(req); + uv_freeaddrinfo(res); + return; + } + + struct sockaddr_in* addr = (struct sockaddr_in*)res->ai_addr; + payload->cb(payload->who, addr->sin_addr.s_addr, payload->port, payload->context); + + c3_free(payload->who); + uv_poll_stop(payload->poll); + c3_free(payload->poll); + uv_freeaddrinfo(res); + c3_free(req); + DNSServiceRefDeallocate(payload->sref); + c3_free(payload); + +} + +static void resolve_cb(DNSServiceRef sref, + DNSServiceFlags f, + uint32_t interface, + DNSServiceErrorType err, + const char *name, + const char *host, + uint16_t port, + uint16_t tl, + const unsigned char *t, + void *context) +{ + mdns_payload* payload = (mdns_payload*)context; + + if (err != kDNSServiceErr_NoError) { + u3l_log("mdns: dns resolve error %d", err); + uv_poll_stop(payload->poll); + c3_free(payload->poll); + c3_free(payload); + DNSServiceRefDeallocate(sref); + return; + } + + payload->sref = sref; + payload->port = port; + + char* who = strtok(name, "."); + + if (who == NULL) { + u3l_log("mdns: invalid domain name"); + uv_poll_stop(payload->poll); + c3_free(payload->poll); + c3_free(payload); + DNSServiceRefDeallocate(sref); + return; + } + + char* who2 = c3_malloc(strlen(who) + 1); + snprintf(who2, sizeof who2, "~%s", who); + payload->who = who2; + + struct addrinfo hints; + memset(&hints, 0, sizeof(hints)); + hints.ai_family = AF_INET; // Request only IPv4 addresses + hints.ai_socktype = SOCK_STREAM; // TCP socket + + uv_getaddrinfo_t* req = (uv_getaddrinfo_t*)c3_malloc(sizeof(uv_getaddrinfo_t)); + req->data = (void*)payload; + + uv_loop_t* loop = uv_default_loop(); + int error = uv_getaddrinfo(loop, req, getaddrinfo_cb, host, NULL, &hints); + + if (error < 0) { + u3l_log("mdns: getaddrinfo error: %s\n", uv_strerror(error)); + uv_poll_stop(payload->poll); + c3_free(payload->poll); + c3_free(payload); + c3_free(req); + DNSServiceRefDeallocate(sref); + } +} + +static void poll_cb(uv_poll_t *handle, int status, int events) { + DNSServiceRef sref = (DNSServiceRef) handle->data; + int err = DNSServiceProcessResult(sref); +} + +static void dns_sd_cb(DNSServiceRef sref, mdns_payload* payload) { + int fd = DNSServiceRefSockFD(sref); + uv_loop_t* loop = uv_default_loop(); + uv_poll_t* poll = (uv_poll_t *)c3_malloc(sizeof(uv_poll_t)); + poll->data = (void *)sref; + payload->poll = poll; + uv_poll_init(loop, poll, fd); + uv_poll_start(poll, UV_READABLE, poll_cb); +} + +static void browse_cb(DNSServiceRef s, + DNSServiceFlags f, + uint32_t interface, + DNSServiceErrorType err, + const char* name, + const char* type, + const char* domain, + void* context) +{ + if (err != kDNSServiceErr_NoError) { + u3l_log("mdns: service browse error %i", err); + return; + } + + if (f & kDNSServiceFlagsAdd) { // Add + DNSServiceRef sr; + + mdns_payload* payload = (mdns_payload*)context; + mdns_payload* payload_copy = c3_malloc(sizeof *payload_copy); + + // copy to prevent asynchronous thrashing of payload + memcpy(payload_copy, payload, sizeof(mdns_payload)); + + DNSServiceErrorType err = + DNSServiceResolve(&sr, 0, interface, + name, type, domain, resolve_cb, + (void*)payload_copy); + + if (err != kDNSServiceErr_NoError) { + u3l_log("mdns: dns service resolve error %i", err); + c3_free(payload_copy); + DNSServiceRefDeallocate(sr); + return; + } + dns_sd_cb(sr, payload_copy); + } +} + +static void register_cb(DNSServiceRef sref, + DNSServiceFlags f, + DNSServiceErrorType err, + const char* name, + const char* type, + const char* domain, + void* context) +{ + mdns_payload* payload = (mdns_payload*)context; + + if (err != kDNSServiceErr_NoError) { + u3l_log("mdns: service register error %i", err); + } else { + u3l_log("mdns: %s registered on all interfaces", name); + } + uv_poll_stop(payload->poll); + c3_free(payload->poll); + c3_free(payload->our); + c3_free(payload); +} + +void mdns_init(uint16_t port, char* our, mdns_cb* cb, void* context) +{ + #if defined(U3_OS_linux) + setenv("AVAHI_COMPAT_NOWARN", "1", 0); + setenv("DBUS_SYSTEM_BUS_ADDRESS", "unix:path=/var/run/dbus/system_bus_socket", 0); + # endif + + memmove(our, our+1, strlen(our)); // certain url parsers don't like the sig + + mdns_payload* register_payload = (mdns_payload*)c3_malloc(sizeof(mdns_payload)); + + register_payload->cb = cb; + register_payload->context = context; + register_payload->our = our; + + DNSServiceRef sref; + DNSServiceErrorType err; + err = DNSServiceRegister(&sref, 0, 0, our, "_ames._udp", + NULL, NULL, htons(port), 0, NULL, register_cb, (void*)register_payload); + + if (err != kDNSServiceErr_NoError) { + u3l_log("mdns: service register error %i", err); + DNSServiceRefDeallocate(sref); + return; + } + + dns_sd_cb(sref, register_payload); + + mdns_payload* browse_payload = (mdns_payload*)c3_malloc(sizeof(mdns_payload)); + + browse_payload->cb = cb; + browse_payload->context = context; + + DNSServiceErrorType dnserr; + DNSServiceRef sref2; + + dnserr = DNSServiceBrowse(&sref2, 0, 0, "_ames._udp", NULL, browse_cb, (void *)browse_payload); + + if (dnserr != kDNSServiceErr_NoError) { + u3l_log("mdns: service browse error %i", dnserr); + DNSServiceRefDeallocate(sref2); + return; + } + + dns_sd_cb(sref2, browse_payload); + +} diff --git a/pkg/vere/vere.h b/pkg/vere/vere.h index cef16810b6..3154432fc5 100644 --- a/pkg/vere/vere.h +++ b/pkg/vere/vere.h @@ -674,6 +674,12 @@ uv_timer_t tim_u; // gc timer } u3_king; + /* mdns_cb: callback when mdns finds a new ship + */ + typedef void mdns_cb(char* ship, + unsigned long s_addr, + uint16_t port, + void* context); /* u3_pier_spin(): (re-)activate idle handler */ void @@ -1238,6 +1244,12 @@ u3_auto* u3_lick_io_init(u3_pier* pir_u); + /** mdns, routing for the local network + **/ + /* mdns_init(): initialize mdns. + */ + void + mdns_init(uint16_t port, char* our, mdns_cb* cb, void* context); /** HTTP server. **/ /* u3_http_io_init(): initialize http I/O. From 3a06580796def107bfa66a52f1b4e450f240d7fe Mon Sep 17 00:00:00 2001 From: Sidnym Ladrut Date: Mon, 4 Sep 2023 23:36:43 +0000 Subject: [PATCH 078/271] add 'mate' jet --- pkg/noun/jets/b/mate.c | 30 ++++++++++++++++++++++++++++++ pkg/noun/jets/q.h | 2 ++ pkg/noun/jets/tree.c | 3 +++ pkg/noun/jets/w.h | 2 ++ 4 files changed, 37 insertions(+) create mode 100644 pkg/noun/jets/b/mate.c diff --git a/pkg/noun/jets/b/mate.c b/pkg/noun/jets/b/mate.c new file mode 100644 index 0000000000..77e4fbb819 --- /dev/null +++ b/pkg/noun/jets/b/mate.c @@ -0,0 +1,30 @@ +/// @file + +#include "jets/q.h" +#include "jets/w.h" + +#include "noun.h" + + + u3_noun + u3qb_mate(u3_noun a, + u3_noun b) + { + if ( u3_nul == b ) { + return u3k(a); + } else if ( u3_nul == a ) { + return u3k(b); + } else if ( c3y == u3r_sing(a, b) ) { + return u3k(a); + } else { + return u3m_bail(c3__fail); + } + } + u3_noun + u3wb_mate(u3_noun cor) + { + u3_noun a, b; + u3x_mean(cor, u3x_sam_2, &a, u3x_sam_3, &b, 0); + return u3qb_mate(a, b); + } + diff --git a/pkg/noun/jets/q.h b/pkg/noun/jets/q.h index 33c63ef42b..8a7c398c3c 100644 --- a/pkg/noun/jets/q.h +++ b/pkg/noun/jets/q.h @@ -32,6 +32,7 @@ u3_noun u3qb_lien(u3_noun, u3_noun); u3_noun u3qb_murn(u3_noun, u3_noun); u3_noun u3qb_need(u3_noun); + u3_noun u3qb_mate(u3_noun, u3_noun); u3_noun u3qb_reap(u3_atom, u3_noun); u3_noun u3qb_reel(u3_noun, u3_noun); u3_noun u3qb_roll(u3_noun, u3_noun); @@ -254,3 +255,4 @@ void u3qf_test(const c3_c*, u3_noun); #endif /* ifndef U3_JETS_Q_H */ + diff --git a/pkg/noun/jets/tree.c b/pkg/noun/jets/tree.c index c15e29ba8e..e2ce880293 100644 --- a/pkg/noun/jets/tree.c +++ b/pkg/noun/jets/tree.c @@ -2139,6 +2139,7 @@ static c3_c* _k140_ha[] = { /* new jets */ + static u3j_harm _139_two_mate_a[] = {{".2", u3wb_mate, c3y}, {}}; static u3j_harm _139_hex_json_de_a[] = {{".2", u3we_json_de}, {}}; static u3j_harm _139_hex_json_en_a[] = {{".2", u3we_json_en}, {}}; static u3j_core _139_hex_json_d[] = @@ -2282,6 +2283,7 @@ static u3j_core _139_two_d[] = { "lien", 7, _140_two_lien_a, 0, no_hashes }, { "murn", 7, _140_two_murn_a, 0, no_hashes }, { "need", 7, _140_two_need_a, 0, no_hashes }, + { "mate", 7, _139_two_mate_a, 0, no_hashes }, { "reap", 7, _140_two_reap_a, 0, no_hashes }, { "reel", 7, _140_two_reel_a, 0, no_hashes }, { "roll", 7, _140_two_roll_a, 0, no_hashes }, @@ -2438,3 +2440,4 @@ u3j_Dash = { 0, 0 }; + diff --git a/pkg/noun/jets/w.h b/pkg/noun/jets/w.h index d838416c03..8b9873c684 100644 --- a/pkg/noun/jets/w.h +++ b/pkg/noun/jets/w.h @@ -32,6 +32,7 @@ u3_noun u3wb_lien(u3_noun); u3_noun u3wb_murn(u3_noun); u3_noun u3wb_need(u3_noun); + u3_noun u3wb_mate(u3_noun); u3_noun u3wb_reap(u3_noun); u3_noun u3wb_reel(u3_noun); u3_noun u3wb_roll(u3_noun); @@ -332,3 +333,4 @@ u3_noun u3wfu_rest(u3_noun); #endif /* ifndef U3_JETS_W_H */ + From 3779c75beb52c72a09c1673faeb45d40b80b6cd4 Mon Sep 17 00:00:00 2001 From: pkova Date: Tue, 5 Sep 2023 13:20:56 +0300 Subject: [PATCH 079/271] bazel: host h2o from s3 since github is unable to serve it reliably --- WORKSPACE.bazel | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/WORKSPACE.bazel b/WORKSPACE.bazel index d1dad3d96a..8fa018b97a 100644 --- a/WORKSPACE.bazel +++ b/WORKSPACE.bazel @@ -207,7 +207,7 @@ versioned_http_archive( patches = ["//bazel/third_party/h2o:{version}.patch"], sha256 = "f8cbc1b530d85ff098f6efc2c3fdbc5e29baffb30614caac59d5c710f7bda201", strip_prefix = "h2o-{version}", - url = "https://github.com/h2o/h2o/archive/refs/tags/v{version}.tar.gz", + url = "https://urbit-foundation.s3.us-east-2.amazonaws.com/h2o-{version}.tar.gz", # When bumping the version, compare `CMakeLists.txt` in the `h2o` repo to # {build_file} and confirm that {build_file} remains an accurate description # of the h2o build process. From b3c381a581623bcedadf759988c64224b820b2bd Mon Sep 17 00:00:00 2001 From: pkova Date: Tue, 5 Sep 2023 13:47:32 +0300 Subject: [PATCH 080/271] bazel: mess with config options to make dbus build in CI --- bazel/third_party/dbus/dbus.BUILD | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bazel/third_party/dbus/dbus.BUILD b/bazel/third_party/dbus/dbus.BUILD index ac55b84103..0642883163 100644 --- a/bazel/third_party/dbus/dbus.BUILD +++ b/bazel/third_party/dbus/dbus.BUILD @@ -13,7 +13,7 @@ configure_make( "//conditions:default": ["--jobs=`nproc`"], }), copts = ["-O3"], - configure_options = ["--disable-selinux"], + configure_options = ["--disable-selinux --without-x --disable-tests"], lib_source = ":all", configure_in_place = True, deps = ["@expat"], From fd96138634a9408cd652c945fe19948270ba2571 Mon Sep 17 00:00:00 2001 From: pkova Date: Tue, 5 Sep 2023 14:58:28 +0300 Subject: [PATCH 081/271] mdns: free our on register_cb fail --- pkg/vere/mdns.c | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/vere/mdns.c b/pkg/vere/mdns.c index 54fd797cf4..77ac270824 100644 --- a/pkg/vere/mdns.c +++ b/pkg/vere/mdns.c @@ -199,6 +199,7 @@ void mdns_init(uint16_t port, char* our, mdns_cb* cb, void* context) if (err != kDNSServiceErr_NoError) { u3l_log("mdns: service register error %i", err); + c3_free(our); DNSServiceRefDeallocate(sref); return; } From 66b17ccaaf89ab918e5aabd1875d4380362c54de Mon Sep 17 00:00:00 2001 From: pkova Date: Tue, 5 Sep 2023 16:25:21 +0300 Subject: [PATCH 082/271] bazel: revert serving h2o from our s3 since github incident is over This reverts commit 3779c75beb52c72a09c1673faeb45d40b80b6cd4. --- WORKSPACE.bazel | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/WORKSPACE.bazel b/WORKSPACE.bazel index 8fa018b97a..d1dad3d96a 100644 --- a/WORKSPACE.bazel +++ b/WORKSPACE.bazel @@ -207,7 +207,7 @@ versioned_http_archive( patches = ["//bazel/third_party/h2o:{version}.patch"], sha256 = "f8cbc1b530d85ff098f6efc2c3fdbc5e29baffb30614caac59d5c710f7bda201", strip_prefix = "h2o-{version}", - url = "https://urbit-foundation.s3.us-east-2.amazonaws.com/h2o-{version}.tar.gz", + url = "https://github.com/h2o/h2o/archive/refs/tags/v{version}.tar.gz", # When bumping the version, compare `CMakeLists.txt` in the `h2o` repo to # {build_file} and confirm that {build_file} remains an accurate description # of the h2o build process. From 43e92073468972b5d9de57f5a1526b4eb1082cda Mon Sep 17 00:00:00 2001 From: pkova Date: Tue, 5 Sep 2023 16:45:17 +0300 Subject: [PATCH 083/271] mdns: fix wonky indentation --- pkg/vere/mdns.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/vere/mdns.c b/pkg/vere/mdns.c index 77ac270824..38bcf3fa77 100644 --- a/pkg/vere/mdns.c +++ b/pkg/vere/mdns.c @@ -55,7 +55,7 @@ static void resolve_cb(DNSServiceRef sref, { mdns_payload* payload = (mdns_payload*)context; - if (err != kDNSServiceErr_NoError) { + if (err != kDNSServiceErr_NoError) { u3l_log("mdns: dns resolve error %d", err); uv_poll_stop(payload->poll); c3_free(payload->poll); @@ -105,11 +105,11 @@ static void resolve_cb(DNSServiceRef sref, static void poll_cb(uv_poll_t *handle, int status, int events) { DNSServiceRef sref = (DNSServiceRef) handle->data; - int err = DNSServiceProcessResult(sref); + int err = DNSServiceProcessResult(sref); } static void dns_sd_cb(DNSServiceRef sref, mdns_payload* payload) { - int fd = DNSServiceRefSockFD(sref); + int fd = DNSServiceRefSockFD(sref); uv_loop_t* loop = uv_default_loop(); uv_poll_t* poll = (uv_poll_t *)c3_malloc(sizeof(uv_poll_t)); poll->data = (void *)sref; @@ -214,7 +214,7 @@ void mdns_init(uint16_t port, char* our, mdns_cb* cb, void* context) DNSServiceErrorType dnserr; DNSServiceRef sref2; - dnserr = DNSServiceBrowse(&sref2, 0, 0, "_ames._udp", NULL, browse_cb, (void *)browse_payload); + dnserr = DNSServiceBrowse(&sref2, 0, 0, "_ames._udp", NULL, browse_cb, (void *)browse_payload); if (dnserr != kDNSServiceErr_NoError) { u3l_log("mdns: service browse error %i", dnserr); From 390735c9d1207885da7f0d1fdd086c9bb9d83e4c Mon Sep 17 00:00:00 2001 From: pkova Date: Tue, 5 Sep 2023 18:42:21 +0300 Subject: [PATCH 084/271] mdns: stop polling as soon as we reach resolve_cb --- pkg/vere/mdns.c | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/pkg/vere/mdns.c b/pkg/vere/mdns.c index 38bcf3fa77..b38e5ea737 100644 --- a/pkg/vere/mdns.c +++ b/pkg/vere/mdns.c @@ -20,8 +20,6 @@ static void getaddrinfo_cb(uv_getaddrinfo_t* req, int status, struct addrinfo* r if (status < 0) { u3l_log("mdns: getaddrinfo error: %s", uv_strerror(status)); - uv_poll_stop(payload->poll); - c3_free(payload->poll); DNSServiceRefDeallocate(payload->sref); c3_free(payload); c3_free(req); @@ -33,8 +31,6 @@ static void getaddrinfo_cb(uv_getaddrinfo_t* req, int status, struct addrinfo* r payload->cb(payload->who, addr->sin_addr.s_addr, payload->port, payload->context); c3_free(payload->who); - uv_poll_stop(payload->poll); - c3_free(payload->poll); uv_freeaddrinfo(res); c3_free(req); DNSServiceRefDeallocate(payload->sref); @@ -55,10 +51,11 @@ static void resolve_cb(DNSServiceRef sref, { mdns_payload* payload = (mdns_payload*)context; + uv_poll_stop(payload->poll); + c3_free(payload->poll); + if (err != kDNSServiceErr_NoError) { u3l_log("mdns: dns resolve error %d", err); - uv_poll_stop(payload->poll); - c3_free(payload->poll); c3_free(payload); DNSServiceRefDeallocate(sref); return; @@ -71,8 +68,6 @@ static void resolve_cb(DNSServiceRef sref, if (who == NULL) { u3l_log("mdns: invalid domain name"); - uv_poll_stop(payload->poll); - c3_free(payload->poll); c3_free(payload); DNSServiceRefDeallocate(sref); return; @@ -95,8 +90,6 @@ static void resolve_cb(DNSServiceRef sref, if (error < 0) { u3l_log("mdns: getaddrinfo error: %s\n", uv_strerror(error)); - uv_poll_stop(payload->poll); - c3_free(payload->poll); c3_free(payload); c3_free(req); DNSServiceRefDeallocate(sref); From fb332c070728e17fc5fcb64355295766d2982213 Mon Sep 17 00:00:00 2001 From: Sidnym Ladrut Date: Wed, 6 Sep 2023 16:33:20 +0000 Subject: [PATCH 085/271] add fixes propsed by ~master-morzod --- pkg/noun/jets/b/mate.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/noun/jets/b/mate.c b/pkg/noun/jets/b/mate.c index 77e4fbb819..f8dbd31654 100644 --- a/pkg/noun/jets/b/mate.c +++ b/pkg/noun/jets/b/mate.c @@ -14,10 +14,10 @@ return u3k(a); } else if ( u3_nul == a ) { return u3k(b); - } else if ( c3y == u3r_sing(a, b) ) { + } else if ( c3y == u3r_sing(u3t(a), u3t(b)) ) { return u3k(a); } else { - return u3m_bail(c3__fail); + return u3m_error("mate"); } } u3_noun From e99d9b50f66d4733fc196804b3d75b41874153a0 Mon Sep 17 00:00:00 2001 From: pkova Date: Fri, 8 Sep 2023 15:26:36 +0300 Subject: [PATCH 086/271] mdns: address joes feedback --- pkg/vere/BUILD.bazel | 1 + pkg/vere/io/ames.c | 16 ++++++++----- pkg/vere/mdns.c | 56 ++++++++++++++++---------------------------- pkg/vere/mdns.h | 5 ++++ pkg/vere/vere.h | 12 ---------- 5 files changed, 36 insertions(+), 54 deletions(-) create mode 100644 pkg/vere/mdns.h diff --git a/pkg/vere/BUILD.bazel b/pkg/vere/BUILD.bazel index 249fc42e42..7e772ae61c 100644 --- a/pkg/vere/BUILD.bazel +++ b/pkg/vere/BUILD.bazel @@ -119,6 +119,7 @@ vere_library( hdrs = [ "db/lmdb.h", "vere.h", + "mdns.h", ] + select({ "@platforms//os:macos": [], "@platforms//os:linux": [ diff --git a/pkg/vere/io/ames.c b/pkg/vere/io/ames.c index 6dd6cff67d..cba1dbf48d 100644 --- a/pkg/vere/io/ames.c +++ b/pkg/vere/io/ames.c @@ -1,6 +1,7 @@ /// @file #include "vere.h" +#include "mdns.h" #include "noun.h" #include "ur.h" @@ -2063,7 +2064,6 @@ _ames_recv_cb(uv_udp_t* wax_u, static void _mdns_dear_bail(u3_ovum* egg_u, u3_noun lud) { - u3l_log("mdns: %%dear failure;"); u3z(lud); u3_ovum_free(egg_u); } @@ -2071,7 +2071,7 @@ _mdns_dear_bail(u3_ovum* egg_u, u3_noun lud) /* _ames_put_dear(): send lane to arvo after hearing mdns response */ static void -_ames_put_dear(char* ship, unsigned long s_addr, uint16_t port, void* context) +_ames_put_dear(c3_c* ship, c3_w s_addr, c3_s port, void* context) { u3_ames* sam_u = (u3_ames*)context; @@ -2088,6 +2088,7 @@ _ames_put_dear(char* ship, unsigned long s_addr, uint16_t port, void* context) u3_noun our = u3i_chubs(2, sam_u->pir_u->who_d); if (our == u3t(shp_u)) { + u3z(shp_u); u3z(our); return; } @@ -2168,11 +2169,14 @@ _ames_io_start(u3_ames* sam_u) u3l_log("ames: live on %d (localhost only)", sam_u->pir_u->por_s); } - u3_noun our = u3dc("scot", 'p', who); - char* our_s = u3r_string(our); - u3z(our); + { + u3_noun our = u3dc("scot", 'p', u3k(who)); + char* our_s = u3r_string(our); + u3z(our); - mdns_init(por_s, our_s, _ames_put_dear, (void *)sam_u); + mdns_init(por_s, our_s, _ames_put_dear, (void *)sam_u); + c3_free(our_s); + } uv_udp_recv_start(&sam_u->wax_u, _ames_alloc, _ames_recv_cb); diff --git a/pkg/vere/mdns.c b/pkg/vere/mdns.c index b38e5ea737..9489ad4841 100644 --- a/pkg/vere/mdns.c +++ b/pkg/vere/mdns.c @@ -1,16 +1,15 @@ /// @file #include "vere.h" - +#include "mdns.h" #include "dns_sd.h" typedef struct _mdns_payload { mdns_cb* cb; DNSServiceRef sref; - char* who; - char* our; + char who[58]; uint16_t port; - uv_poll_t* poll; + uv_poll_t poll; void* context; } mdns_payload; @@ -30,7 +29,6 @@ static void getaddrinfo_cb(uv_getaddrinfo_t* req, int status, struct addrinfo* r struct sockaddr_in* addr = (struct sockaddr_in*)res->ai_addr; payload->cb(payload->who, addr->sin_addr.s_addr, payload->port, payload->context); - c3_free(payload->who); uv_freeaddrinfo(res); c3_free(req); DNSServiceRefDeallocate(payload->sref); @@ -51,8 +49,7 @@ static void resolve_cb(DNSServiceRef sref, { mdns_payload* payload = (mdns_payload*)context; - uv_poll_stop(payload->poll); - c3_free(payload->poll); + uv_poll_stop(&payload->poll); if (err != kDNSServiceErr_NoError) { u3l_log("mdns: dns resolve error %d", err); @@ -64,19 +61,13 @@ static void resolve_cb(DNSServiceRef sref, payload->sref = sref; payload->port = port; - char* who = strtok(name, "."); - - if (who == NULL) { - u3l_log("mdns: invalid domain name"); - c3_free(payload); - DNSServiceRefDeallocate(sref); - return; + int i; + payload->who[0] = '~'; + for (i = 0; name[i] != '\0' && name[i] != '.' && i < 58; ++i) + { + payload->who[i+1] = name[i]; } - char* who2 = c3_malloc(strlen(who) + 1); - snprintf(who2, sizeof who2, "~%s", who); - payload->who = who2; - struct addrinfo hints; memset(&hints, 0, sizeof(hints)); hints.ai_family = AF_INET; // Request only IPv4 addresses @@ -96,19 +87,17 @@ static void resolve_cb(DNSServiceRef sref, } } -static void poll_cb(uv_poll_t *handle, int status, int events) { +static void poll_cb(uv_poll_t* handle, int status, int events) { DNSServiceRef sref = (DNSServiceRef) handle->data; int err = DNSServiceProcessResult(sref); } -static void dns_sd_cb(DNSServiceRef sref, mdns_payload* payload) { +static void init_sref_poll(DNSServiceRef sref, mdns_payload* payload) { int fd = DNSServiceRefSockFD(sref); uv_loop_t* loop = uv_default_loop(); - uv_poll_t* poll = (uv_poll_t *)c3_malloc(sizeof(uv_poll_t)); - poll->data = (void *)sref; - payload->poll = poll; - uv_poll_init(loop, poll, fd); - uv_poll_start(poll, UV_READABLE, poll_cb); + payload->poll.data = (void*)sref; + uv_poll_init(loop, &payload->poll, fd); + uv_poll_start(&payload->poll, UV_READABLE, poll_cb); } static void browse_cb(DNSServiceRef s, @@ -145,7 +134,7 @@ static void browse_cb(DNSServiceRef s, DNSServiceRefDeallocate(sr); return; } - dns_sd_cb(sr, payload_copy); + init_sref_poll(sr, payload_copy); } } @@ -164,9 +153,9 @@ static void register_cb(DNSServiceRef sref, } else { u3l_log("mdns: %s registered on all interfaces", name); } - uv_poll_stop(payload->poll); - c3_free(payload->poll); - c3_free(payload->our); + // not freeing sref with DNSServiceRefDeallocate since that + // deregisters us from mdns + uv_poll_stop(&payload->poll); c3_free(payload); } @@ -181,10 +170,6 @@ void mdns_init(uint16_t port, char* our, mdns_cb* cb, void* context) mdns_payload* register_payload = (mdns_payload*)c3_malloc(sizeof(mdns_payload)); - register_payload->cb = cb; - register_payload->context = context; - register_payload->our = our; - DNSServiceRef sref; DNSServiceErrorType err; err = DNSServiceRegister(&sref, 0, 0, our, "_ames._udp", @@ -192,12 +177,11 @@ void mdns_init(uint16_t port, char* our, mdns_cb* cb, void* context) if (err != kDNSServiceErr_NoError) { u3l_log("mdns: service register error %i", err); - c3_free(our); DNSServiceRefDeallocate(sref); return; } - dns_sd_cb(sref, register_payload); + init_sref_poll(sref, register_payload); mdns_payload* browse_payload = (mdns_payload*)c3_malloc(sizeof(mdns_payload)); @@ -215,6 +199,6 @@ void mdns_init(uint16_t port, char* our, mdns_cb* cb, void* context) return; } - dns_sd_cb(sref2, browse_payload); + init_sref_poll(sref2, browse_payload); } diff --git a/pkg/vere/mdns.h b/pkg/vere/mdns.h new file mode 100644 index 0000000000..66c3477179 --- /dev/null +++ b/pkg/vere/mdns.h @@ -0,0 +1,5 @@ +#include "noun.h" + +typedef void mdns_cb(c3_c* ship, c3_w s_addr, c3_s port, void* context); + +void mdns_init(uint16_t port, char* our, mdns_cb* cb, void* context); diff --git a/pkg/vere/vere.h b/pkg/vere/vere.h index 3154432fc5..cef16810b6 100644 --- a/pkg/vere/vere.h +++ b/pkg/vere/vere.h @@ -674,12 +674,6 @@ uv_timer_t tim_u; // gc timer } u3_king; - /* mdns_cb: callback when mdns finds a new ship - */ - typedef void mdns_cb(char* ship, - unsigned long s_addr, - uint16_t port, - void* context); /* u3_pier_spin(): (re-)activate idle handler */ void @@ -1244,12 +1238,6 @@ u3_auto* u3_lick_io_init(u3_pier* pir_u); - /** mdns, routing for the local network - **/ - /* mdns_init(): initialize mdns. - */ - void - mdns_init(uint16_t port, char* our, mdns_cb* cb, void* context); /** HTTP server. **/ /* u3_http_io_init(): initialize http I/O. From 7aaa3a9c8464e5dd5db6f0e449ca18f388678b09 Mon Sep 17 00:00:00 2001 From: pkova Date: Mon, 11 Sep 2023 13:41:11 +0300 Subject: [PATCH 087/271] bazel: remove unused options from expat.BUILD --- bazel/third_party/expat/expat.BUILD | 2 -- 1 file changed, 2 deletions(-) diff --git a/bazel/third_party/expat/expat.BUILD b/bazel/third_party/expat/expat.BUILD index 10e536a620..030f81ed78 100644 --- a/bazel/third_party/expat/expat.BUILD +++ b/bazel/third_party/expat/expat.BUILD @@ -11,10 +11,8 @@ configure_make( "@platforms//os:macos": ["--jobs=`sysctl -n hw.logicalcpu`"], "//conditions:default": ["--jobs=`nproc`"], }), - # configure_options = ["--with-xml=none --disable-libevent --disable-glib --disable-gobject --disable-gdbm --disable-qt3 --disable-qt4 --disable-qt5 --disable-gtk --disable-gtk3 --disable-mono --disable-monodoc --disable-python --enable-compat-libdns_sd"], copts = ["-O3"], lib_source = ":all", - # deps = ["@dbus"], out_static_libs = ["libexpat.a"], visibility = ["//visibility:public"], ) From 4ac9875fd8472e188da92bae8eea95973a243f88 Mon Sep 17 00:00:00 2001 From: pkova Date: Mon, 11 Sep 2023 13:41:58 +0300 Subject: [PATCH 088/271] ames: shp_u -> whu in _ames_put_dear --- pkg/vere/io/ames.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pkg/vere/io/ames.c b/pkg/vere/io/ames.c index cba1dbf48d..03f316a6a7 100644 --- a/pkg/vere/io/ames.c +++ b/pkg/vere/io/ames.c @@ -2079,16 +2079,16 @@ _ames_put_dear(c3_c* ship, c3_w s_addr, c3_s port, void* context) lan.pip_w = ntohl(s_addr); lan.por_s = ntohs(port); - u3_noun shp_u = u3dc("slaw", c3__p, u3i_string(ship)); + u3_noun whu = u3dc("slaw", c3__p, u3i_string(ship)); - if (u3_nul == shp_u) { + if (u3_nul == whu) { u3l_log("ames: strange ship from mdns: %s", ship); return; } u3_noun our = u3i_chubs(2, sam_u->pir_u->who_d); - if (our == u3t(shp_u)) { - u3z(shp_u); + if (our == u3t(whu)) { + u3z(whu); u3z(our); return; } @@ -2096,13 +2096,13 @@ _ames_put_dear(c3_c* ship, c3_w s_addr, c3_s port, void* context) u3z(our); u3_noun wir = u3nc(c3__ames, u3_nul); - u3_noun cad = u3nt(c3__dear, u3k(u3t(shp_u)), u3nc(c3n, u3_ames_encode_lane(lan))); + u3_noun cad = u3nt(c3__dear, u3k(u3t(whu)), u3nc(c3n, u3_ames_encode_lane(lan))); u3_auto_peer( u3_auto_plan(&sam_u->car_u, u3_ovum_init(0, c3__a, wir, cad)), 0, 0, _mdns_dear_bail); - u3z(shp_u); + u3z(whu); } /* _ames_io_start(): initialize ames I/O. From b173a028b6d7293516e15bdd0d310dca1fd5c68c Mon Sep 17 00:00:00 2001 From: pkova Date: Mon, 11 Sep 2023 14:12:11 +0300 Subject: [PATCH 089/271] mdns: use pointer arithmetic instead of memmove to desig --- pkg/vere/mdns.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/vere/mdns.c b/pkg/vere/mdns.c index 9489ad4841..62b07a8219 100644 --- a/pkg/vere/mdns.c +++ b/pkg/vere/mdns.c @@ -77,6 +77,7 @@ static void resolve_cb(DNSServiceRef sref, req->data = (void*)payload; uv_loop_t* loop = uv_default_loop(); + int error = uv_getaddrinfo(loop, req, getaddrinfo_cb, host, NULL, &hints); if (error < 0) { @@ -166,13 +167,12 @@ void mdns_init(uint16_t port, char* our, mdns_cb* cb, void* context) setenv("DBUS_SYSTEM_BUS_ADDRESS", "unix:path=/var/run/dbus/system_bus_socket", 0); # endif - memmove(our, our+1, strlen(our)); // certain url parsers don't like the sig - mdns_payload* register_payload = (mdns_payload*)c3_malloc(sizeof(mdns_payload)); DNSServiceRef sref; DNSServiceErrorType err; - err = DNSServiceRegister(&sref, 0, 0, our, "_ames._udp", + char* our_no_sig = our + 1; // certain url parsers don't like the ~ + err = DNSServiceRegister(&sref, 0, 0, our_no_sig, "_ames._udp", NULL, NULL, htons(port), 0, NULL, register_cb, (void*)register_payload); if (err != kDNSServiceErr_NoError) { From 0f0f56223076b42ab0263ba5c7ee1c1c5886006c Mon Sep 17 00:00:00 2001 From: Matthew LeVan Date: Fri, 1 Sep 2023 16:42:34 -0400 Subject: [PATCH 090/271] loom: compile-time pointer compression with migration --- pkg/noun/BUILD.bazel | 5 +- pkg/noun/allocate.c | 51 +++----- pkg/noun/allocate.h | 70 ++++------- pkg/noun/hashtable.c | 14 --- pkg/noun/hashtable.h | 4 +- pkg/noun/jets.c | 1 - pkg/noun/manage.c | 222 +++++----------------------------- pkg/noun/manage.h | 10 +- pkg/noun/options.h | 8 -- pkg/noun/v1/allocate.c | 261 ++++++++++++++++++++++++++++++++++++++++ pkg/noun/v1/allocate.h | 78 ++++++++++++ pkg/noun/v1/hashtable.c | 173 ++++++++++++++++++++++++++ pkg/noun/v1/hashtable.h | 51 ++++++++ pkg/noun/v1/jets.c | 101 ++++++++++++++++ pkg/noun/v1/jets.h | 33 +++++ pkg/noun/v1/manage.c | 20 +++ pkg/noun/v1/manage.h | 17 +++ pkg/noun/v1/nock.c | 70 +++++++++++ pkg/noun/v1/nock.h | 24 ++++ pkg/noun/v1/vortex.c | 17 +++ pkg/noun/v1/vortex.h | 21 ++++ pkg/noun/v2/allocate.c | 73 +++++++++++ pkg/noun/v2/allocate.h | 133 ++++++++++++++++++++ pkg/noun/v2/hashtable.c | 143 ++++++++++++++++++++++ pkg/noun/v2/hashtable.h | 41 +++++++ pkg/noun/v2/jets.c | 37 ++++++ pkg/noun/v2/jets.h | 22 ++++ pkg/noun/v2/manage.c | 177 +++++++++++++++++++++++++++ pkg/noun/v2/manage.h | 14 +++ pkg/noun/v2/nock.c | 28 +++++ pkg/noun/v2/nock.h | 18 +++ pkg/noun/v2/options.h | 14 +++ pkg/noun/v2/vortex.c | 25 ++++ pkg/noun/v2/vortex.h | 40 ++++++ pkg/noun/vortex.h | 3 - 35 files changed, 1711 insertions(+), 308 deletions(-) create mode 100644 pkg/noun/v1/allocate.c create mode 100644 pkg/noun/v1/allocate.h create mode 100644 pkg/noun/v1/hashtable.c create mode 100644 pkg/noun/v1/hashtable.h create mode 100644 pkg/noun/v1/jets.c create mode 100644 pkg/noun/v1/jets.h create mode 100644 pkg/noun/v1/manage.c create mode 100644 pkg/noun/v1/manage.h create mode 100644 pkg/noun/v1/nock.c create mode 100644 pkg/noun/v1/nock.h create mode 100644 pkg/noun/v1/vortex.c create mode 100644 pkg/noun/v1/vortex.h create mode 100644 pkg/noun/v2/allocate.c create mode 100644 pkg/noun/v2/allocate.h create mode 100644 pkg/noun/v2/hashtable.c create mode 100644 pkg/noun/v2/hashtable.h create mode 100644 pkg/noun/v2/jets.c create mode 100644 pkg/noun/v2/jets.h create mode 100644 pkg/noun/v2/manage.c create mode 100644 pkg/noun/v2/manage.h create mode 100644 pkg/noun/v2/nock.c create mode 100644 pkg/noun/v2/nock.h create mode 100644 pkg/noun/v2/options.h create mode 100644 pkg/noun/v2/vortex.c create mode 100644 pkg/noun/v2/vortex.h diff --git a/pkg/noun/BUILD.bazel b/pkg/noun/BUILD.bazel index 964e4bf60f..1949a7d878 100644 --- a/pkg/noun/BUILD.bazel +++ b/pkg/noun/BUILD.bazel @@ -8,9 +8,8 @@ vere_library( name = "noun", srcs = glob( [ - "*.c", - "*.h", - "jets/tree.c", + "**/*.c", + "**/*.h", "jets/*.h", "jets/**/*.c", ], diff --git a/pkg/noun/allocate.c b/pkg/noun/allocate.c index ace4473a4a..44f7175879 100644 --- a/pkg/noun/allocate.c +++ b/pkg/noun/allocate.c @@ -16,15 +16,6 @@ u3_road* u3a_Road; c3_w u3_Code; #endif -// declarations of inline functions -// - -void u3a_config_loom(c3_w ver_w); -void *u3a_into(c3_w x); -c3_w u3a_outa(void *p); -c3_w u3a_to_off(c3_w som); -void *u3a_to_ptr(c3_w som); -c3_w *u3a_to_wtr(c3_w som); c3_w u3a_to_pug(c3_w off); c3_w u3a_to_pom(c3_w off); @@ -67,12 +58,12 @@ _box_count(c3_ws siz_ws) { } _box_count, (others?) should have perhaps its own header and certainly its own prefix. having to remind yourself that _box_count doesn't actually do anything unless U3_CPU_DEBUG is defined is annoying. */ -#define _box_vaal(box_u) \ - do { \ - c3_dessert(((uintptr_t)u3a_boxto(box_u) \ - & u3C.balign_d-1) == 0); \ - c3_dessert((((u3a_box*)(box_u))->siz_w \ - & u3C.walign_w-1) == 0); \ +#define _box_vaal(box_u) \ + do { \ + c3_dessert(((uintptr_t)u3a_boxto(box_u) \ + & u3a_balign-1) == 0); \ + c3_dessert((((u3a_box*)(box_u))->siz_w \ + & u3a_walign-1) == 0); \ } while(0) /* _box_slot(): select the right free list to search for a block. @@ -302,7 +293,7 @@ _ca_box_make_hat(c3_w len_w, c3_w ald_w, c3_w off_w, c3_w use_w) all_p += c3_wiseof(u3a_box) + off_w; pad_w = c3_align(all_p, ald_w, C3_ALGHI) - all_p; - siz_w = c3_align(len_w + pad_w, u3C.walign_w, C3_ALGHI); + siz_w = c3_align(len_w + pad_w, u3a_walign, C3_ALGHI); // hand-inlined: siz_w >= u3a_open(u3R) // @@ -316,7 +307,7 @@ _ca_box_make_hat(c3_w len_w, c3_w ald_w, c3_w off_w, c3_w use_w) all_p += c3_wiseof(u3a_box) + off_w; pad_w = all_p - c3_align(all_p, ald_w, C3_ALGLO); - siz_w = c3_align(len_w + pad_w, u3C.walign_w, C3_ALGHI); + siz_w = c3_align(len_w + pad_w, u3a_walign, C3_ALGHI); // hand-inlined: siz_w >= u3a_open(u3R) // @@ -508,7 +499,7 @@ _ca_willoc(c3_w len_w, c3_w ald_w, c3_w off_w) box_p = all_p = *pfr_p; all_p += c3_wiseof(u3a_box) + off_w; c3_w pad_w = c3_align(all_p, ald_w, C3_ALGHI) - all_p; - c3_w des_w = c3_align(siz_w + pad_w, u3C.walign_w, C3_ALGHI); + c3_w des_w = c3_align(siz_w + pad_w, u3a_walign, C3_ALGHI); /* calls maximally requesting DWORD alignment of returned pointer shouldn't require padding. */ @@ -711,17 +702,17 @@ u3a_wtrim(void* tox_v, c3_w old_w, c3_w len_w) c3_w* box_w = (void*)u3a_botox(nov_w); c3_w* end_w = c3_align(nov_w + len_w + 1, /* +1 for trailing allocation size */ - u3C.balign_d, + u3a_balign, C3_ALGHI); c3_w asz_w = (end_w - box_w); /* total size in words of new allocation */ if (box_u->siz_w <= asz_w) return; c3_w bsz_w = box_u->siz_w - asz_w; /* size diff in words between old and new */ - c3_dessert(asz_w && ((asz_w & u3C.walign_w-1) == 0)); /* new allocation size must be non-zero and DWORD multiple */ + c3_dessert(asz_w && ((asz_w & u3a_walign-1) == 0)); /* new allocation size must be non-zero and DWORD multiple */ c3_dessert(end_w < (box_w + box_u->siz_w)); /* desired alloc end must not exceed existing boundaries */ - c3_dessert(((uintptr_t)end_w & u3C.balign_d-1) == 0); /* address of box getting freed must be DWORD aligned */ - c3_dessert((bsz_w & u3C.walign_w-1) == 0); /* size of box getting freed must be DWORD multiple */ + c3_dessert(((uintptr_t)end_w & u3a_balign-1) == 0); /* address of box getting freed must be DWORD aligned */ + c3_dessert((bsz_w & u3a_walign-1) == 0); /* size of box getting freed must be DWORD multiple */ _box_attach(_box_make(end_w, bsz_w, 0)); /* free the unneeded space */ @@ -1747,24 +1738,12 @@ u3a_rewritten_noun(u3_noun som) return som; } u3_post som_p = u3a_rewritten(u3a_to_off(som)); - - /* If this is being called during a migration, one-bit pointer compression - needs to be temporarily enabled so the rewritten reference is compressed */ - if (u3C.migration_state == MIG_REWRITE_COMPRESSED) - u3C.vits_w = 1; - if ( c3y == u3a_is_pug(som) ) { - som_p = u3a_to_pug(som_p); + return u3a_to_pug(som_p); } else { - som_p = u3a_to_pom(som_p); + return u3a_to_pom(som_p); } - - /* likewise, pointer compression is disabled until migration is complete */ - if (u3C.migration_state == MIG_REWRITE_COMPRESSED) - u3C.vits_w = 0; - - return som_p; } /* u3a_mark_mptr(): mark a malloc-allocated ptr for gc. diff --git a/pkg/noun/allocate.h b/pkg/noun/allocate.h index 7355954abd..be50793002 100644 --- a/pkg/noun/allocate.h +++ b/pkg/noun/allocate.h @@ -3,7 +3,6 @@ #include "error.h" #include "manage.h" -#include "options.h" /** Constants. **/ @@ -11,14 +10,21 @@ */ # define u3a_bits U3_OS_LoomBits /* 30 */ - /* u3a_vits_max: number of virtual bits in a reference gained via pointer - compression + /* u3a_vits: number of virtual bits in a noun reference gained via shifting */ -# define u3a_vits_max 1 +# define u3a_vits 1 + + /* u3a_walign: references into the loom are guaranteed to be word-aligned to: + */ +# define u3a_walign (1 << u3a_vits) + + /* u3a_balign: u3a_walign in bytes + */ +# define u3a_balign (sizeof(c3_w)*u3a_walign) /* u3a_bits_max: max loom bex */ -# define u3a_bits_max (8 * sizeof(c3_w) + u3a_vits_max) +# define u3a_bits_max (8 * sizeof(c3_w) + u3a_vits) /* u3a_page: number of bits in word-addressed page. 12 == 16K page */ @@ -26,11 +32,11 @@ /* u3a_pages: maximum number of pages in memory. */ -# define u3a_pages (1ULL << (u3a_bits + u3a_vits_max - u3a_page) ) +# define u3a_pages (1ULL << (u3a_bits + u3a_vits - u3a_page) ) /* u3a_words: maximum number of words in memory. */ -# define u3a_words ( 1ULL << (u3a_bits + u3a_vits_max )) +# define u3a_words ( 1ULL << (u3a_bits + u3a_vits)) /* u3a_bytes: maximum number of bytes in memory. */ @@ -355,7 +361,7 @@ # define _rod_vaal(rod_u) \ do { \ c3_dessert(((uintptr_t)((u3a_road*)(rod_u))->hat_p \ - & u3C.walign_w-1) == 0); \ + & u3a_walign-1) == 0); \ } while(0) @@ -374,68 +380,40 @@ # define u3_Loom ((c3_w *)(void *)U3_OS_LoomBase) - /** inline functions. + /** Inline functions. **/ - /* u3a_config_loom(): configure loom information by u3v version - */ - inline void u3a_config_loom(c3_w ver_w) { - switch (ver_w) { - case U3V_VER1: - u3C.vits_w = 0; - break; - case U3V_VER2: - u3C.vits_w = 1; - break; - default: - u3_assert(0); - } - - u3C.walign_w = 1 << u3C.vits_w; - u3C.balign_d = sizeof(c3_w) * u3C.walign_w; -} - /* u3a_into(): convert loom offset [x] into generic pointer. */ - inline void *u3a_into(c3_w x) { - return u3_Loom + x; - } +# define u3a_into(x) ((void *)(u3_Loom + (x))) /* u3a_outa(): convert pointer [p] into word offset into loom. */ - inline c3_w u3a_outa(void *p) { - return ((c3_w *)p) - u3_Loom; - } +# define u3a_outa(p) ((c3_w *)(void *)(p) - u3_Loom) /* u3a_to_off(): mask off bits 30 and 31 from noun [som]. */ - inline c3_w u3a_to_off(c3_w som) { - return (som & 0x3fffffff) << u3C.vits_w; - } +# define u3a_to_off(som) (((som) & 0x3fffffff) << u3a_vits) /* u3a_to_ptr(): convert noun [som] into generic pointer into loom. */ - inline void *u3a_to_ptr(c3_w som) { - return u3a_into(u3a_to_off(som)); - } +# define u3a_to_ptr(som) (u3a_into(u3a_to_off(som))) /* u3a_to_wtr(): convert noun [som] into word pointer into loom. */ - inline c3_w *u3a_to_wtr(c3_w som) { - return (c3_w *)u3a_to_ptr(som); - } +# define u3a_to_wtr(som) ((c3_w *)u3a_to_ptr(som)) /* u3a_to_pug(): set bit 31 of [off]. */ inline c3_w u3a_to_pug(c3_w off) { - c3_dessert((off & u3C.walign_w-1) == 0); - return (off >> u3C.vits_w) | 0x80000000; + c3_dessert((off & u3a_walign-1) == 0); + return (off >> u3a_vits) | 0x80000000; } /* u3a_to_pom(): set bits 30 and 31 of [off]. */ inline c3_w u3a_to_pom(c3_w off) { - c3_dessert((off & u3C.walign_w-1) == 0); - return (off >> u3C.vits_w) | 0xc0000000; + c3_dessert((off & u3a_walign-1) == 0); + return (off >> u3a_vits) | 0xc0000000; } /** road stack. diff --git a/pkg/noun/hashtable.c b/pkg/noun/hashtable.c index 5915de5944..bc5d751b8c 100644 --- a/pkg/noun/hashtable.c +++ b/pkg/noun/hashtable.c @@ -1017,15 +1017,8 @@ _ch_rewrite_node(u3h_node* han_u, c3_w lef_w) else { void* hav_v = u3h_slot_to_node(sot_w); u3h_node* nod_u = u3to(u3h_node,u3a_rewritten(u3of(u3h_node,hav_v))); - - if (u3C.migration_state == MIG_REWRITE_COMPRESSED) - u3C.vits_w = 1; - han_u->sot_w[i_w] = u3h_node_to_slot(nod_u); - if (u3C.migration_state == MIG_REWRITE_COMPRESSED) - u3C.vits_w = 0; - if ( 0 == lef_w ) { _ch_rewrite_buck(hav_v); } else { @@ -1057,15 +1050,8 @@ u3h_rewrite(u3p(u3h_root) har_p) else if ( _(u3h_slot_is_node(sot_w)) ) { u3h_node* han_u = u3h_slot_to_node(sot_w); u3h_node* nod_u = u3to(u3h_node,u3a_rewritten(u3of(u3h_node,han_u))); - - if (u3C.migration_state == MIG_REWRITE_COMPRESSED) - u3C.vits_w = 1; - har_u->sot_w[i_w] = u3h_node_to_slot(nod_u); - if (u3C.migration_state == MIG_REWRITE_COMPRESSED) - u3C.vits_w = 0; - _ch_rewrite_node(han_u, 25); } } diff --git a/pkg/noun/hashtable.h b/pkg/noun/hashtable.h index be1d3f93be..a117f50011 100644 --- a/pkg/noun/hashtable.h +++ b/pkg/noun/hashtable.h @@ -79,8 +79,8 @@ # define u3h_slot_is_node(sot) ((1 == ((sot) >> 30)) ? c3y : c3n) # define u3h_slot_is_noun(sot) ((1 == ((sot) >> 31)) ? c3y : c3n) # define u3h_slot_is_warm(sot) (((sot) & 0x40000000) ? c3y : c3n) -# define u3h_slot_to_node(sot) (u3a_into(((sot) & 0x3fffffff) << u3C.vits_w)) -# define u3h_node_to_slot(ptr) ((u3a_outa((ptr)) >> u3C.vits_w) | 0x40000000) +# define u3h_slot_to_node(sot) (u3a_into(((sot) & 0x3fffffff) << u3a_vits)) +# define u3h_node_to_slot(ptr) ((u3a_outa((ptr)) >> u3a_vits) | 0x40000000) # define u3h_noun_be_warm(sot) ((sot) | 0x40000000) # define u3h_noun_be_cold(sot) ((sot) & ~0x40000000) # define u3h_slot_to_noun(sot) (0x40000000 | (sot)) diff --git a/pkg/noun/jets.c b/pkg/noun/jets.c index 00edefec40..dcabcdb559 100644 --- a/pkg/noun/jets.c +++ b/pkg/noun/jets.c @@ -2377,7 +2377,6 @@ u3j_reclaim(void) // if ( &(u3H->rod_u) == u3R ) { // u3j_ream(); // } - // clear the jet hank cache // u3h_walk(u3R->jed.han_p, _cj_free_hank); diff --git a/pkg/noun/manage.c b/pkg/noun/manage.c index 5ff7d7b129..60787bde34 100644 --- a/pkg/noun/manage.c +++ b/pkg/noun/manage.c @@ -488,7 +488,7 @@ _pave_parts(void) static u3_road* _pave_road(c3_w* rut_w, c3_w* mat_w, c3_w* cap_w, c3_w siz_w) { - c3_dessert(((uintptr_t)rut_w & u3C.balign_d-1) == 0); + c3_dessert(((uintptr_t)rut_w & u3a_balign-1) == 0); u3_road* rod_u = (void*) mat_w; // enable in case of corruption @@ -529,8 +529,8 @@ _pave_north(c3_w* mem_w, c3_w siz_w, c3_w len_w, c3_o kid_o) // 00~~~|R|---|H|######|C|+++|M|~~~FF // ^--u3R which _pave_road returns (u3H for home road) // - c3_w* mat_w = c3_align(mem_w + len_w - siz_w, u3C.balign_d, C3_ALGLO); - c3_w* rut_w = c3_align(mem_w, u3C.balign_d, C3_ALGHI); + c3_w* mat_w = c3_align(mem_w + len_w - siz_w, u3a_balign, C3_ALGLO); + c3_w* rut_w = c3_align(mem_w, u3a_balign, C3_ALGHI); c3_w* cap_w = mat_w; if ( c3y == kid_o ) { @@ -560,8 +560,8 @@ _pave_south(c3_w* mem_w, c3_w siz_w, c3_w len_w) // 00~~~|M|+++|C|######|H|---|R|~~~FFF // ^---u3R which _pave_road returns // - c3_w* mat_w = c3_align(mem_w, u3C.balign_d, C3_ALGHI); - c3_w* rut_w = c3_align(mem_w + len_w, u3C.balign_d, C3_ALGLO); + c3_w* mat_w = c3_align(mem_w, u3a_balign, C3_ALGHI); + c3_w* rut_w = c3_align(mem_w + len_w, u3a_balign, C3_ALGLO); c3_w* cap_w = mat_w + siz_w; u3e_ward(u3of(c3_w, cap_w) - 1, u3of(c3_w, rut_w)); @@ -574,12 +574,9 @@ _pave_south(c3_w* mem_w, c3_w siz_w, c3_w len_w) static void _pave_home(void) { - /* a pristine home road will always have compressed references */ - u3a_config_loom(U3V_VERLAT); - - c3_w* mem_w = u3_Loom + u3C.walign_w; + c3_w* mem_w = u3_Loom + u3a_walign; c3_w siz_w = c3_wiseof(u3v_home); - c3_w len_w = u3C.wor_i - u3C.walign_w; + c3_w len_w = u3C.wor_i - u3a_walign; u3H = (void *)_pave_north(mem_w, siz_w, len_w, c3n); u3H->ver_w = U3V_VERLAT; @@ -597,14 +594,24 @@ static void _find_home(void) { c3_w ver_w = *(u3_Loom + u3C.wor_i - 1); - u3a_config_loom(ver_w); + + switch ( ver_w ) { + case 1: u3m_v2_migrate(); + case 2: break; + default: { + fprintf(stderr, "loom: checkpoint version mismatch: " + "have %u, need %u\r\n", + ver_w, U3V_VERLAT); + abort(); + } + } // NB: the home road is always north // - c3_w* mem_w = u3_Loom + u3C.walign_w; + c3_w* mem_w = u3_Loom + u3a_walign; c3_w siz_w = c3_wiseof(u3v_home); - c3_w len_w = u3C.wor_i - u3C.walign_w; - c3_w* mat_w = c3_align(mem_w + len_w - siz_w, u3C.balign_d, C3_ALGLO); + c3_w len_w = u3C.wor_i - u3a_walign; + c3_w* mat_w = c3_align(mem_w + len_w - siz_w, u3a_balign, C3_ALGLO); u3H = (void *)mat_w; u3R = &u3H->rod_u; @@ -642,18 +649,6 @@ _find_home(void) /* As a further guard against any sneaky loom corruption */ u3a_loom_sane(); - if (U3V_VERLAT > ver_w) { - u3m_migrate(U3V_VERLAT); - u3a_config_loom(U3V_VERLAT); - } - else if ( U3V_VERLAT < ver_w ) { - fprintf(stderr, "loom: checkpoint version mismatch: " - "have %u, need %u\r\n", - ver_w, - U3V_VERLAT); - abort(); - } - _rod_vaal(u3R); } @@ -748,13 +743,6 @@ u3m_dump(void) c3_i u3m_bail(u3_noun how) { - if ( &(u3H->rod_u) == u3R ) { - // XX set exit code - // - fprintf(stderr, "home: bailing out\r\n"); - abort(); - } - // printf some metadata // switch ( how ) { @@ -779,6 +767,13 @@ u3m_bail(u3_noun how) } } + if ( &(u3H->rod_u) == u3R ) { + // XX set exit code + // + fprintf(stderr, "home: bailing out\r\n"); + abort(); + } + // intercept fatal errors // switch ( how ) { @@ -865,7 +860,7 @@ u3m_leap(c3_w pad_w) } pad_w += c3_wiseof(u3a_road); len_w = u3a_open(u3R) - pad_w; - c3_align(len_w, u3C.walign_w, C3_ALGHI); + c3_align(len_w, u3a_walign, C3_ALGHI); } /* Allocate a region on the cap. @@ -2174,7 +2169,7 @@ u3m_boot_lite(size_t len_i) return 0; } -/* u3m_reclaim: clear persistent caches to reclaim memory +/* u3m_reclaim: clear persistent caches to reclaim memory. */ void u3m_reclaim(void) @@ -2190,7 +2185,7 @@ u3m_reclaim(void) static void _cm_pack_rewrite(void) { - // XX fix u3a_rewrit* to support south roads + // XX fix u3a_rewrite* to support south roads // u3_assert( &(u3H->rod_u) == u3R ); @@ -2228,158 +2223,3 @@ u3m_pack(void) return (u3a_open(u3R) - pre_w); } - -static void -_migrate_reclaim() -{ - fprintf(stderr, "loom: migration reclaim\r\n"); - u3m_reclaim(); -} - -static void -_migrate_seek(const u3a_road *rod_u) -{ - /* - very much like u3a_pack_seek with the following changes: - - there is no need to account for free space as |pack is performed before - the migration - - odd sized boxes will be padded by one word to achieve an even size - - rut will be moved from one word ahead of u3_Loom to two words ahead - */ - c3_w * box_w = u3a_into(rod_u->rut_p); - c3_w * end_w = u3a_into(rod_u->hat_p); - u3_post new_p = (rod_u->rut_p + 1 + c3_wiseof(u3a_box)); - u3a_box * box_u = (void *)box_w; - - fprintf(stderr, "loom: migration seek\r\n"); - - for (; box_w < end_w - ; box_w += box_u->siz_w - , box_u = (void*)box_w) - { - if (!box_u->use_w) - continue; - u3_assert(box_u->siz_w); - u3_assert(box_u->use_w); - box_w[box_u->siz_w - 1] = new_p; - new_p = c3_align(new_p + box_u->siz_w, 2, C3_ALGHI); - } -} - -static void -_migrate_rewrite() -{ - fprintf(stderr, "loom: migration rewrite\r\n"); - - /* So that rewritten pointers are compressed, this flag is set */ - u3C.migration_state = MIG_REWRITE_COMPRESSED; - _cm_pack_rewrite(); - u3C.migration_state = MIG_NONE; -} - -static void -_migrate_move(u3a_road *rod_u) -{ - fprintf(stderr, "loom: migration move\r\n"); - - c3_z hiz_z = u3a_heap(rod_u) * sizeof(c3_w); - - /* calculate required shift distance to prevent write head overlapping read head */ - c3_w off_w = 1; /* at least 1 word because u3R->rut_p migrates from 1 to 2 */ - for (u3a_box *box_u = u3a_into(rod_u->rut_p) - ; (void *)box_u < u3a_into(rod_u->hat_p) - ; box_u = (void *)((c3_w *)box_u + box_u->siz_w)) - off_w += box_u->siz_w & 1; /* odd-sized boxes are padded by one word */ - - /* shift */ - memmove(u3a_into(u3H->rod_u.rut_p + off_w), - u3a_into(u3H->rod_u.rut_p), - hiz_z); - /* manually zero the former rut */ - *(c3_w *)u3a_into(rod_u->rut_p) = 0; - - /* relocate boxes to DWORD-aligned addresses stored in trailing size word */ - c3_w *box_w = u3a_into(rod_u->rut_p + off_w); - c3_w *end_w = u3a_into(rod_u->hat_p + off_w); - u3a_box *old_u = (void *)box_w; - c3_w siz_w = old_u->siz_w; - u3p(c3_w) new_p = rod_u->rut_p + 1 + c3_wiseof(u3a_box); - c3_w *new_w; - - for (; box_w < end_w - ; box_w += siz_w - , old_u = (void *)box_w - , siz_w = old_u->siz_w) { - old_u->use_w &= 0x7fffffff; - - if (!old_u->use_w) - continue; - - new_w = (void *)u3a_botox(u3a_into(new_p)); - u3_assert(box_w[siz_w - 1] == new_p); - u3_assert(new_w <= box_w); - - c3_w i_w; - for (i_w = 0; i_w < siz_w - 1; i_w++) - new_w[i_w] = box_w[i_w]; - - if (siz_w & 1) { - new_w[i_w++] = 0; /* pad odd sized boxes */ - new_w[i_w++] = siz_w + 1; /* restore trailing size word */ - new_w[0] = siz_w + 1; /* and the leading size word */ - } - else { - new_w[i_w++] = siz_w; - } - - new_p += i_w; - } - - /* restore proper heap state */ - rod_u->rut_p = 2; - rod_u->hat_p = new_p - c3_wiseof(u3a_box); - - /* like |pack, clear the free lists and cell allocator */ - for (c3_w i_w = 0; i_w < u3a_fbox_no; i_w++) - u3R->all.fre_p[i_w] = 0; - - u3R->all.fre_w = 0; - u3R->all.cel_p = 0; -} - - -/* u3m_migrate: perform loom migration if necessary. - ver_w - target version -*/ -void -u3m_migrate(u3v_version ver_w) -{ - if (u3H->ver_w == ver_w) - return; - - /* 1 -> 2 is all that is currently supported */ - c3_dessert(u3H->ver_w == U3V_VER1 && - ver_w == U3V_VER2); - - /* only home road migration is supported */ - c3_dessert((uintptr_t)u3H == (uintptr_t)u3R); - - fprintf(stderr, "loom: migration running. This may take several minutes to perform.\r\n"); - fprintf(stderr, "loom: have version: %"PRIc3_w" migrating to version: %"PRIc3_w"\r\n", - u3H->ver_w, ver_w); - - /* packing first simplifies migration logic and minimizes required buffer space */ - u3m_pack(); - - /* perform the migration in a pattern similar to |pack */ - _migrate_reclaim(); - _migrate_seek(&u3H->rod_u); - _migrate_rewrite(); - _migrate_move(&u3H->rod_u); - - /* finally update the version and commit to disk */ - u3H->ver_w = ver_w; - /* extra assurance we haven't corrupted the loom before writing to disk */ - u3a_loom_sane(); - u3m_save(); -} diff --git a/pkg/noun/manage.h b/pkg/noun/manage.h index 297585c486..bc8ff97390 100644 --- a/pkg/noun/manage.h +++ b/pkg/noun/manage.h @@ -3,6 +3,9 @@ #ifndef U3_MANAGE_H #define U3_MANAGE_H +#include "v1/manage.h" +#include "v2/manage.h" + #include "c3.h" #include "types.h" #include "version.h" @@ -180,7 +183,7 @@ void u3m_wall(u3_noun wol); - /* u3m_reclaim: clear persistent caches to reclaim memory + /* u3m_reclaim: clear persistent caches to reclaim memory. */ void u3m_reclaim(void); @@ -190,10 +193,9 @@ c3_w u3m_pack(void); - /* u3m_migrate: perform loom migration if necessary. - ver_w - target version + /* u3m_migrate: perform loom migrations. */ void - u3m_migrate(u3v_version ver_w); + u3m_migrate(); #endif /* ifndef U3_MANAGE_H */ diff --git a/pkg/noun/options.h b/pkg/noun/options.h index 1bd65320fc..52732d2655 100644 --- a/pkg/noun/options.h +++ b/pkg/noun/options.h @@ -15,14 +15,6 @@ c3_c* dir_c; // execution directory (pier) c3_c* eph_c; // ephemeral file c3_w wag_w; // flags (both ways) - c3_w vits_w; // number of virtual bits in reference - c3_w walign_w; // word alignment - c3_d balign_d; // byte alignment - enum { - MIG_NONE, - MIG_REWRITE_COMPRESSED, - } migration_state; - size_t wor_i; // loom word-length (<= u3a_words) c3_w tos_w; // loom toss skip-length void (*stderr_log_f)(c3_c*); // errors from c code diff --git a/pkg/noun/v1/allocate.c b/pkg/noun/v1/allocate.c new file mode 100644 index 0000000000..226aa33f00 --- /dev/null +++ b/pkg/noun/v1/allocate.c @@ -0,0 +1,261 @@ +/// @file + +#include "pkg/noun/allocate.h" +#include "pkg/noun/v1/allocate.h" + +#include "pkg/noun/v1/hashtable.h" + +/* _box_v1_slot(): select the right free list to search for a block. +*/ +static c3_w +_box_v1_slot(c3_w siz_w) +{ + if ( siz_w < u3a_v1_minimum ) { + return 0; + } + else { + c3_w i_w = 1; + + while ( 1 ) { + if ( i_w == u3a_v1_fbox_no ) { + return (i_w - 1); + } + if ( siz_w < 16 ) { + return i_w; + } + siz_w = (siz_w + 1) >> 1; + i_w += 1; + } + } +} + +/* _box_v1_make(): construct a box. +*/ +static u3a_v1_box* +_box_v1_make(void* box_v, c3_w siz_w, c3_w use_w) +{ + u3a_v1_box* box_u = box_v; + c3_w* box_w = box_v; + + u3_assert(siz_w >= u3a_v1_minimum); + + box_w[0] = siz_w; + box_w[siz_w - 1] = siz_w; + box_u->use_w = use_w; + +# ifdef U3_MEMORY_DEBUG + box_u->cod_w = u3_Code; + box_u->eus_w = 0; +# endif + + return box_u; +} + +/* _box_v1_attach(): attach a box to the free list. +*/ +static void +_box_v1_attach(u3a_v1_box* box_u) +{ + u3_assert(box_u->siz_w >= (1 + c3_wiseof(u3a_v1_fbox))); + u3_assert(0 != u3of(u3a_v1_fbox, box_u)); + +#if 0 + // For debugging, fill the box with beef. + { + c3_w* box_w = (void *)box_u; + c3_w i_w; + + for ( i_w = c3_wiseof(u3a_v1_box); (i_w + 1) < box_u->siz_w; i_w++ ) { + box_w[i_w] = 0xdeadbeef; + } + } +#endif + + { + c3_w sel_w = _box_v1_slot(box_u->siz_w); + u3p(u3a_v1_fbox) fre_p = u3of(u3a_v1_fbox, box_u); + u3p(u3a_v1_fbox)* pfr_p = &u3R_v1->all.fre_p[sel_w]; + u3p(u3a_v1_fbox) nex_p = *pfr_p; + + u3to(u3a_v1_fbox, fre_p)->pre_p = 0; + u3to(u3a_v1_fbox, fre_p)->nex_p = nex_p; + if ( u3to(u3a_v1_fbox, fre_p)->nex_p ) { + u3to(u3a_v1_fbox, u3to(u3a_v1_fbox, fre_p)->nex_p)->pre_p = fre_p; + } + (*pfr_p) = fre_p; + } +} + +/* _box_v1_detach(): detach a box from the free list. +*/ +static void +_box_v1_detach(u3a_v1_box* box_u) +{ + u3p(u3a_v1_fbox) fre_p = u3of(u3a_v1_fbox, box_u); + u3p(u3a_v1_fbox) pre_p = u3to(u3a_v1_fbox, fre_p)->pre_p; + u3p(u3a_v1_fbox) nex_p = u3to(u3a_v1_fbox, fre_p)->nex_p; + + + if ( nex_p ) { + if ( u3to(u3a_v1_fbox, nex_p)->pre_p != fre_p ) { + u3_assert(!"loom: corrupt"); + } + u3to(u3a_v1_fbox, nex_p)->pre_p = pre_p; + } + if ( pre_p ) { + if( u3to(u3a_v1_fbox, pre_p)->nex_p != fre_p ) { + u3_assert(!"loom: corrupt"); + } + u3to(u3a_v1_fbox, pre_p)->nex_p = nex_p; + } + else { + c3_w sel_w = _box_v1_slot(box_u->siz_w); + + if ( fre_p != u3R_v1->all.fre_p[sel_w] ) { + u3_assert(!"loom: corrupt"); + } + u3R_v1->all.fre_p[sel_w] = nex_p; + } +} + +/* _box_v1_free(): free and coalesce. +*/ +static void +_box_v1_free(u3a_v1_box* box_u) +{ + c3_w* box_w = (c3_w *)(void *)box_u; + + u3_assert(box_u->use_w != 0); + box_u->use_w -= 1; + if ( 0 != box_u->use_w ) { + return; + } + +#if 0 + /* Clear the contents of the block, for debugging. + */ + { + c3_w i_w; + + for ( i_w = c3_wiseof(u3a_v1_box); (i_w + 1) < box_u->siz_w; i_w++ ) { + box_w[i_w] = 0xdeadbeef; + } + } +#endif + + if ( c3y == u3a_v1_is_north(u3R_v1) ) { + /* Try to coalesce with the block below. + */ + if ( box_w != u3a_v1_into(u3R_v1->rut_p) ) { + c3_w laz_w = *(box_w - 1); + u3a_v1_box* pox_u = (u3a_v1_box*)(void *)(box_w - laz_w); + + if ( 0 == pox_u->use_w ) { + _box_v1_detach(pox_u); + _box_v1_make(pox_u, (laz_w + box_u->siz_w), 0); + + box_u = pox_u; + box_w = (c3_w*)(void *)pox_u; + } + } + + /* Try to coalesce with the block above, or the wilderness. + */ + if ( (box_w + box_u->siz_w) == u3a_v1_into(u3R_v1->hat_p) ) { + u3R_v1->hat_p = u3a_v1_outa(box_w); + } + else { + u3a_v1_box* nox_u = (u3a_v1_box*)(void *)(box_w + box_u->siz_w); + + if ( 0 == nox_u->use_w ) { + _box_v1_detach(nox_u); + _box_v1_make(box_u, (box_u->siz_w + nox_u->siz_w), 0); + } + _box_v1_attach(box_u); + } + } +} + +/* u3a_v1_wfree(): free storage. +*/ +void +u3a_v1_wfree(void* tox_v) +{ + _box_v1_free(u3a_v1_botox(tox_v)); +} + +/* u3a_v1_free(): free for aligned malloc. +*/ +void +u3a_v1_free(void* tox_v) +{ + if (NULL == tox_v) + return; + + c3_w* tox_w = tox_v; + c3_w pad_w = tox_w[-1]; + c3_w* org_w = tox_w - (pad_w + 1); + + // u3l_log("free %p %p", org_w, tox_w); + u3a_v1_wfree(org_w); +} + +/* u3a_v1_reclaim(): clear ad-hoc persistent caches to reclaim memory. +*/ +void +u3a_v1_reclaim(void) +{ + // clear the memoization cache + // + u3h_v1_free_nodes(u3R_v1->cax.har_p); +} + +/* _me_lose_north(): lose on a north road. +*/ +static void +_me_lose_north(u3_noun dog) +{ +top: + { + c3_w* dog_w = u3a_v1_to_ptr(dog); + u3a_v1_box* box_u = u3a_v1_botox(dog_w); + + if ( box_u->use_w > 1 ) { + box_u->use_w -= 1; + } + else { + if ( 0 == box_u->use_w ) { + u3m_v1_bail(c3__foul); + } + else { + if ( _(u3a_v1_is_pom(dog)) ) { + u3a_v1_cell* dog_u = (void *)dog_w; + u3_noun h_dog = dog_u->hed; + u3_noun t_dog = dog_u->tel; + + if ( !_(u3a_v1_is_cat(h_dog)) ) { + _me_lose_north(h_dog); + } + u3a_v1_wfree(dog_w); + if ( !_(u3a_v1_is_cat(t_dog)) ) { + dog = t_dog; + goto top; + } + } + else { + u3a_v1_wfree(dog_w); + } + } + } + } +} + +/* u3a_v1_lose(): lose a reference count. +*/ +void +u3a_v1_lose(u3_noun som) +{ + if ( !_(u3a_v1_is_cat(som)) ) { + _me_lose_north(som); + } +} diff --git a/pkg/noun/v1/allocate.h b/pkg/noun/v1/allocate.h new file mode 100644 index 0000000000..4544417b78 --- /dev/null +++ b/pkg/noun/v1/allocate.h @@ -0,0 +1,78 @@ +#ifndef U3_ALLOCATE_V1_H +#define U3_ALLOCATE_V1_H + +#include "pkg/noun/allocate.h" +#include "pkg/noun/v2/allocate.h" + + /** Aliases. + **/ +# define u3R_v1 u3R_v2 +# define u3a_v1_botox u3a_v2_botox +# define u3a_v1_box u3a_v2_box +# define u3a_v1_cell u3a_v2_cell +# define u3a_v1_fbox u3a_v2_fbox +# define u3a_v1_fbox_no u3a_v2_fbox_no +# define u3a_v1_into u3a_v2_into +# define u3a_v1_is_cat u3a_v2_is_cat +# define u3a_v1_is_north u3a_v2_is_north +# define u3a_v1_is_pom u3a_v2_is_pom +# define u3a_v1_minimum u3a_v2_minimum +# define u3a_v1_outa u3a_v2_outa + + /** Structures. + **/ + /** Macros. Should be better commented. + **/ + /* Inside a noun. + */ + /* u3a_v1_to_off(): mask off bits 30 and 31 from noun [som]. + */ +# define u3a_v1_to_off(som) ((som) & 0x3fffffff) + + /* u3a_v1_to_ptr(): convert noun [som] into generic pointer into loom. + */ +# define u3a_v1_to_ptr(som) (u3a_v1_into(u3a_v1_to_off(som))) + + /* u3a_v1_to_wtr(): convert noun [som] into word pointer into loom. + */ +# define u3a_v1_to_wtr(som) ((c3_w *)u3a_v1_to_ptr(som)) + + /* u3a_v1_to_pug(): set bit 31 of [off]. + */ +# define u3a_v1_to_pug(off) (off | 0x80000000) + + /* u3a_v1_to_pom(): set bits 30 and 31 of [off]. + */ +# define u3a_v1_to_pom(off) (off | 0xc0000000) + + /** Functions. + **/ + /** Allocation. + **/ + /* Word-aligned allocation. + */ + /* u3a_v1_wfree(): free storage. + */ + void + u3a_v1_wfree(void* lag_v); + + /* C-style aligned allocation - *not* compatible with above. + */ + /* u3a_v1_free(): free for aligned malloc. + */ + void + u3a_v1_free(void* tox_v); + + /* Reference and arena control. + */ + /* u3a_v1_reclaim(): clear ad-hoc persistent caches to reclaim memory. + */ + void + u3a_v1_reclaim(void); + + /* u3a_v1_lose(): lose a reference count. + */ + void + u3a_v1_lose(u3_noun som); + +#endif /* ifndef U3_ALLOCATE_H */ diff --git a/pkg/noun/v1/hashtable.c b/pkg/noun/v1/hashtable.c new file mode 100644 index 0000000000..aa3a9b7aef --- /dev/null +++ b/pkg/noun/v1/hashtable.c @@ -0,0 +1,173 @@ +/// @file + +#include "pkg/noun/hashtable.h" +#include "pkg/noun/v1/hashtable.h" + +#include "pkg/noun/allocate.h" +#include "pkg/noun/v1/allocate.h" + + +/* _ch_v1_popcount(): number of bits set in word. A standard intrinsic. +** NB: copy of _ch_v1_popcount in pkg/noun/hashtable.c +*/ +static c3_w +_ch_v1_popcount(c3_w num_w) +{ + return __builtin_popcount(num_w); +} + +/* _ch_v1_free_buck(): free bucket +** NB: copy of _ch_v1_free_buck in pkg/noun/hashtable.c +*/ +static void +_ch_v1_free_buck(u3h_v1_buck* hab_u) +{ + c3_w i_w; + + for ( i_w = 0; i_w < hab_u->len_w; i_w++ ) { + u3a_v1_lose(u3h_v1_slot_to_noun(hab_u->sot_w[i_w])); + } + u3a_v1_wfree(hab_u); +} + +/* _ch_v1_free_node(): free node. +*/ +static void +_ch_v1_free_node(u3h_v1_node* han_u, c3_w lef_w) +{ + c3_w len_w = _ch_v1_popcount(han_u->map_w); + c3_w i_w; + + lef_w -= 5; + + for ( i_w = 0; i_w < len_w; i_w++ ) { + c3_w sot_w = han_u->sot_w[i_w]; + + if ( _(u3h_v1_slot_is_noun(sot_w)) ) { + u3a_v1_lose(u3h_v1_slot_to_noun(sot_w)); + } + else { + void* hav_v = u3h_v1_slot_to_node(sot_w); + + if ( 0 == lef_w ) { + _ch_v1_free_buck(hav_v); + } else { + _ch_v1_free_node(hav_v, lef_w); + } + } + } + u3a_v1_wfree(han_u); +} + +/* u3h_v1_free_nodes(): free hashtable nodes. +*/ +void +u3h_v1_free_nodes(u3p(u3h_v1_root) har_p) +{ + u3h_v1_root* har_u = u3to(u3h_v1_root, har_p); + c3_w i_w; + + for ( i_w = 0; i_w < 64; i_w++ ) { + c3_w sot_w = har_u->sot_w[i_w]; + + if ( _(u3h_v1_slot_is_noun(sot_w)) ) { + u3a_v1_lose(u3h_v1_slot_to_noun(sot_w)); + } + else if ( _(u3h_v1_slot_is_node(sot_w)) ) { + u3h_v1_node* han_u = (u3h_v1_node*) u3h_v1_slot_to_node(sot_w); + + _ch_v1_free_node(han_u, 25); + } + har_u->sot_w[i_w] = 0; + } + har_u->use_w = 0; + har_u->arm_u.mug_w = 0; + har_u->arm_u.inx_w = 0; +} + +/* _ch_v1_walk_buck(): walk bucket for gc. +** NB: copy of _ch_v1_walk_buck in pkg/noun/hashtable.c +*/ +static void +_ch_v1_walk_buck(u3h_v1_buck* hab_u, void (*fun_f)(u3_noun, void*), void* wit) +{ + c3_w i_w; + + for ( i_w = 0; i_w < hab_u->len_w; i_w++ ) { + fun_f(u3h_v1_slot_to_noun(hab_u->sot_w[i_w]), wit); + } +} + +/* _ch_v1_walk_node(): walk node for gc. +*/ +static void +_ch_v1_walk_node(u3h_v1_node* han_u, c3_w lef_w, void (*fun_f)(u3_noun, void*), void* wit) +{ + c3_w len_w = _ch_v1_popcount(han_u->map_w); + c3_w i_w; + + lef_w -= 5; + + for ( i_w = 0; i_w < len_w; i_w++ ) { + c3_w sot_w = han_u->sot_w[i_w]; + + if ( _(u3h_v1_slot_is_noun(sot_w)) ) { + u3_noun kev = u3h_v1_slot_to_noun(sot_w); + + fun_f(kev, wit); + } + else { + void* hav_v = u3h_v1_slot_to_node(sot_w); + + if ( 0 == lef_w ) { + _ch_v1_walk_buck(hav_v, fun_f, wit); + } else { + _ch_v1_walk_node(hav_v, lef_w, fun_f, wit); + } + } + } +} + +/* u3h_v1_walk_with(): traverse hashtable with key, value fn and data + * argument; RETAINS. +*/ +void +u3h_v1_walk_with(u3p(u3h_v1_root) har_p, + void (*fun_f)(u3_noun, void*), + void* wit) +{ + u3h_v1_root* har_u = u3to(u3h_v1_root, har_p); + c3_w i_w; + + for ( i_w = 0; i_w < 64; i_w++ ) { + c3_w sot_w = har_u->sot_w[i_w]; + + if ( _(u3h_v1_slot_is_noun(sot_w)) ) { + u3_noun kev = u3h_v1_slot_to_noun(sot_w); + + fun_f(kev, wit); + } + else if ( _(u3h_v1_slot_is_node(sot_w)) ) { + u3h_v1_node* han_u = (u3h_v1_node*) u3h_v1_slot_to_node(sot_w); + + _ch_v1_walk_node(han_u, 25, fun_f, wit); + } + } +} + +/* _ch_v1_walk_plain(): use plain u3_noun fun_f for each node + */ +static void +_ch_v1_walk_plain(u3_noun kev, void* wit) +{ + void (*fun_f)(u3_noun) = wit; + fun_f(kev); +} + +/* u3h_v1_walk(): u3h_v1_walk_with, but with no data argument +*/ +void +u3h_v1_walk(u3p(u3h_v1_root) har_p, void (*fun_f)(u3_noun)) +{ + u3h_v1_walk_with(har_p, _ch_v1_walk_plain, fun_f); +} diff --git a/pkg/noun/v1/hashtable.h b/pkg/noun/v1/hashtable.h new file mode 100644 index 0000000000..2d5fdea6f3 --- /dev/null +++ b/pkg/noun/v1/hashtable.h @@ -0,0 +1,51 @@ +#ifndef U3_HASHTABLE_V1_H +#define U3_HASHTABLE_V1_H + +#include "pkg/noun/hashtable.h" +#include "pkg/noun/v2/hashtable.h" + + /** Aliases. + **/ +# define u3h_v1_buck u3h_v2_buck +# define u3h_v1_node u3h_v2_node +# define u3h_v1_root u3h_v2_root +# define u3h_v1_slot_is_node u3h_v2_slot_is_node +# define u3h_v1_slot_is_noun u3h_v2_slot_is_noun +# define u3h_v1_slot_to_noun u3h_v2_slot_to_noun + + /** Data structures. + **/ + + /** HAMT macros. + *** + *** Coordinate with u3_noun definition! + **/ + /* u3h_v1_slot_to_node(): slot to node pointer + ** u3h_v1_node_to_slot(): node pointer to slot + */ +# define u3h_v1_slot_to_node(sot) (u3a_v1_into((sot) & 0x3fffffff)) +# define u3h_v1_node_to_slot(ptr) (u3a_v1_outa(ptr) | 0x40000000) + + /** Functions. + *** + *** Needs: delete and merge functions; clock reclamation function. + **/ + /* u3h_v1_free(): free hashtable. + */ + void + u3h_v1_free_nodes(u3p(u3h_root) har_p); + + /* u3h_v1_walk_with(): traverse hashtable with key, value fn and data + * argument; RETAINS. + */ + void + u3h_v1_walk_with(u3p(u3h_root) har_p, + void (*fun_f)(u3_noun, void*), + void* wit); + + /* u3h_v1_walk(): u3h_v1_walk_with, but with no data argument + */ + void + u3h_v1_walk(u3p(u3h_root) har_p, void (*fun_f)(u3_noun)); + +#endif /* ifndef U3_HASHTABLE_H */ diff --git a/pkg/noun/v1/jets.c b/pkg/noun/v1/jets.c new file mode 100644 index 0000000000..be13a63d49 --- /dev/null +++ b/pkg/noun/v1/jets.c @@ -0,0 +1,101 @@ +/// @file + +#include "pkg/noun/vortex.h" + +#include "pkg/noun/jets.h" +#include "pkg/noun/v1/jets.h" +#include "pkg/noun/v2/jets.h" + +#include "pkg/noun/v1/allocate.h" +#include "pkg/noun/v1/hashtable.h" + +/** Data structures. +**/ + +/* _cj_v1_hank: cached hook information. +** NB: copy of _cj_hank from pkg/noun/jets.c + */ +typedef struct { + u3_weak hax; // axis of hooked inner core + u3j_v1_site sit_u; // call-site data +} _cj_v1_hank; + +/** Functions. +**/ + +/* _cj_fink_free(): lose and free everything in a u3j_v1_fink. +*/ +static void +_cj_v1_fink_free(u3p(u3j_v1_fink) fin_p) +{ + c3_w i_w; + u3j_v1_fink* fin_u = u3to(u3j_v1_fink, fin_p); + u3a_v1_lose(fin_u->sat); + for ( i_w = 0; i_w < fin_u->len_w; ++i_w ) { + u3j_v1_fist* fis_u = &(fin_u->fis_u[i_w]); + u3a_v1_lose(fis_u->bat); + u3a_v1_lose(fis_u->pax); + } + u3a_v1_wfree(fin_u); +} + +/* u3j_v1_rite_lose(): lose references of u3j_v1_rite (but do not free). + */ +void +u3j_v1_rite_lose(u3j_v1_rite* rit_u) +{ + if ( (c3y == rit_u->own_o) && u3_none != rit_u->clu ) { + u3a_v1_lose(rit_u->clu); + _cj_v1_fink_free(rit_u->fin_p); + } +} + + +/* u3j_v1_site_lose(): lose references of u3j_v1_site (but do not free). + */ +void +u3j_v1_site_lose(u3j_v1_site* sit_u) +{ + u3a_v1_lose(sit_u->axe); + if ( u3_none != sit_u->bat ) { + u3a_v1_lose(sit_u->bat); + } + if ( u3_none != sit_u->bas ) { + u3a_v1_lose(sit_u->bas); + } + if ( u3_none != sit_u->loc ) { + u3a_v1_lose(sit_u->loc); + u3a_v1_lose(sit_u->lab); + if ( c3y == sit_u->fon_o ) { + if ( sit_u->fin_p ) { + _cj_v1_fink_free(sit_u->fin_p); + } + } + } +} + +/* _cj_v1_free_hank(): free an entry from the hank cache. +** NB: copy of _cj_v1_free_hank() from pkg/noun/jets.c +*/ +static void +_cj_v1_free_hank(u3_noun kev) +{ + u3a_v1_cell* cel_u = (u3a_v1_cell*) u3a_v1_to_ptr(kev); + _cj_v1_hank* han_u = u3to(_cj_v1_hank, cel_u->tel); + if ( u3_none != han_u->hax ) { + u3a_v1_lose(han_u->hax); + u3j_v1_site_lose(&(han_u->sit_u)); + } + u3a_v1_wfree(han_u); +} + +/* u3j_v1_reclaim(): clear ad-hoc persistent caches to reclaim memory. +*/ +void +u3j_v1_reclaim(void) +{ + // clear the jet hank cache + // + u3h_v1_walk(u3R_v1->jed.han_p, _cj_v1_free_hank); + u3h_v1_free_nodes(u3R_v1->jed.han_p); +} diff --git a/pkg/noun/v1/jets.h b/pkg/noun/v1/jets.h new file mode 100644 index 0000000000..1711a0e675 --- /dev/null +++ b/pkg/noun/v1/jets.h @@ -0,0 +1,33 @@ +/// @file + +#ifndef U3_JETS_V1_H +#define U3_JETS_V1_H + +#include "pkg/noun/jets.h" +#include "pkg/noun/v2/jets.h" + + /** Aliases. + **/ +# define u3j_v1_fink u3j_v2_fink +# define u3j_v1_fist u3j_v2_fist +# define u3j_v1_rite u3j_v2_rite +# define u3j_v1_site u3j_v2_site + + /** Functions. + **/ + /* u3j_v1_reclaim(): clear ad-hoc persistent caches to reclaim memory. + */ + void + u3j_v1_reclaim(void); + + /* u3j_v1_rite_lose(): lose references of u3j_rite (but do not free). + */ + void + u3j_v1_rite_lose(u3j_rite* rit_u); + + /* u3j_v1_site_lose(): lose references of u3j_site (but do not free). + */ + void + u3j_v1_site_lose(u3j_site* sit_u); + +#endif /* ifndef U3_JETS_V1_H */ diff --git a/pkg/noun/v1/manage.c b/pkg/noun/v1/manage.c new file mode 100644 index 0000000000..0eac6a8d7b --- /dev/null +++ b/pkg/noun/v1/manage.c @@ -0,0 +1,20 @@ +/// @file + +#include "pkg/noun/v1/manage.h" + +#include "pkg/noun/v1/allocate.h" +#include "pkg/noun/v1/hashtable.h" +#include "pkg/noun/v1/jets.h" +#include "pkg/noun/v1/nock.h" +#include "pkg/noun/v1/vortex.h" + +/* u3m_v1_reclaim: clear persistent caches to reclaim memory +*/ +void +u3m_v1_reclaim(void) +{ + u3v_v1_reclaim(); + u3j_v1_reclaim(); + u3n_v1_reclaim(); + u3a_v1_reclaim(); +} diff --git a/pkg/noun/v1/manage.h b/pkg/noun/v1/manage.h new file mode 100644 index 0000000000..669d69ea36 --- /dev/null +++ b/pkg/noun/v1/manage.h @@ -0,0 +1,17 @@ +/// @file + +#ifndef U3_MANAGE_V1_H +#define U3_MANAGE_V1_H + + /** Aliases. + **/ +# define u3m_v1_bail u3m_bail + + /** System management. + **/ + /* u3m_v1_reclaim: clear persistent caches to reclaim memory + */ + void + u3m_v1_reclaim(void); + +#endif /* ifndef U3_MANAGE_V1_H */ diff --git a/pkg/noun/v1/nock.c b/pkg/noun/v1/nock.c new file mode 100644 index 0000000000..4e0de27dc2 --- /dev/null +++ b/pkg/noun/v1/nock.c @@ -0,0 +1,70 @@ +/// @file + +#include "pkg/noun/nock.h" +#include "pkg/noun/v1/nock.h" + +#include "pkg/noun/v1/allocate.h" +#include "pkg/noun/v1/hashtable.h" +#include "pkg/noun/jets.h" +#include "pkg/noun/v1/jets.h" + +/* u3n_v1_reclaim(): clear ad-hoc persistent caches to reclaim memory. +*/ +void +u3n_v1_reclaim(void) +{ + // clear the bytecode cache + // + // We can't just u3h_v1_free() -- the value is a post to a u3n_v1_prog. + // Note that the hank cache *must* also be freed (in u3j_v1_reclaim()) + // + u3n_v1_free(); +} + +/* _cn_v1_prog_free(): free memory retained by program pog_u +** NB: copy of _cn_v1_prog_free in pkg/noun/nock.c +*/ +static void +_cn_v1_prog_free(u3n_v1_prog* pog_u) +{ + // fix up pointers for loom portability + // pog_u->byc_u.ops_y = (c3_y*) ((void*) pog_u) + sizeof(u3n_v1_prog); + // pog_u->lit_u.non = (u3_noun*) (pog_u->byc_u.ops_y + pog_u->byc_u.len_w); + // pog_u->mem_u.sot_u = (u3n_memo*) (pog_u->lit_u.non + pog_u->lit_u.len_w); + // pog_u->cal_u.sit_u = (u3j_site*) (pog_u->mem_u.sot_u + pog_u->mem_u.len_w); + // pog_u->reg_u.rit_u = (u3j_rite*) (pog_u->cal_u.sit_u + pog_u->cal_u.len_w); + + c3_w dex_w; + for (dex_w = 0; dex_w < pog_u->lit_u.len_w; ++dex_w) { + u3a_v1_lose(pog_u->lit_u.non[dex_w]); + } + for (dex_w = 0; dex_w < pog_u->mem_u.len_w; ++dex_w) { + u3a_v1_lose(pog_u->mem_u.sot_u[dex_w].key); + } + for (dex_w = 0; dex_w < pog_u->cal_u.len_w; ++dex_w) { + u3j_v1_site_lose(&(pog_u->cal_u.sit_u[dex_w])); + } + for (dex_w = 0; dex_w < pog_u->reg_u.len_w; ++dex_w) { + u3j_v1_rite_lose(&(pog_u->reg_u.rit_u[dex_w])); + } + u3a_v1_free(pog_u); +} + +/* _n_feb(): u3h_v1_walk helper for u3n_v1_free + */ +static void +_n_feb(u3_noun kev) +{ + u3a_v1_cell *cel_u = (u3a_v1_cell*) u3a_v1_to_ptr(kev); + _cn_v1_prog_free(u3to(u3n_v1_prog, cel_u->tel)); +} + +/* u3n_v1_free(): free bytecode cache + */ +void +u3n_v1_free() +{ + u3p(u3h_v1_root) har_p = u3R_v1->byc.har_p; + u3h_v1_walk(har_p, _n_feb); + u3h_v1_free_nodes(har_p); +} diff --git a/pkg/noun/v1/nock.h b/pkg/noun/v1/nock.h new file mode 100644 index 0000000000..08a7a5959a --- /dev/null +++ b/pkg/noun/v1/nock.h @@ -0,0 +1,24 @@ +/// @file + +#ifndef U3_NOCK_V1_H +#define U3_NOCK_V1_H + +#include "pkg/noun/v2/nock.h" + + /** Aliases. + **/ +# define u3n_v1_prog u3n_v2_prog + + /** Functions. + **/ + /* u3n_v1_reclaim(): clear ad-hoc persistent caches to reclaim memory. + */ + void + u3n_v1_reclaim(void); + + /* u3n_v1_free(): free bytecode cache. + */ + void + u3n_v1_free(void); + +#endif /* ifndef U3_NOCK_V1_H */ diff --git a/pkg/noun/v1/vortex.c b/pkg/noun/v1/vortex.c new file mode 100644 index 0000000000..e1ec21b22d --- /dev/null +++ b/pkg/noun/v1/vortex.c @@ -0,0 +1,17 @@ +/// @file + +#include "pkg/noun/vortex.h" +#include "pkg/noun/v1/vortex.h" + +#include "pkg/noun/v1/allocate.h" + +/* u3v_v1_reclaim(): clear ad-hoc persistent caches to reclaim memory. +*/ +void +u3v_v1_reclaim(void) +{ + // clear the u3v_wish cache + // + u3a_v1_lose(u3A_v1->yot); + u3A_v1->yot = u3_nul; +} diff --git a/pkg/noun/v1/vortex.h b/pkg/noun/v1/vortex.h new file mode 100644 index 0000000000..f6c0b223d6 --- /dev/null +++ b/pkg/noun/v1/vortex.h @@ -0,0 +1,21 @@ +/// @file + +#ifndef U3_VORTEX_V1_H +#define U3_VORTEX_V1_H + +#include "pkg/noun/allocate.h" +#include "pkg/noun/v2/vortex.h" + + /** Aliases. + **/ +# define u3H_v1 u3H_v2 +# define u3A_v1 u3A_v2 + + /** Functions. + **/ + /* u3v_v1_reclaim(): clear ad-hoc persistent caches to reclaim memory. + */ + void + u3v_v1_reclaim(void); + +#endif /* ifndef U3_VORTEX_V1_H */ diff --git a/pkg/noun/v2/allocate.c b/pkg/noun/v2/allocate.c new file mode 100644 index 0000000000..dfae7b8213 --- /dev/null +++ b/pkg/noun/v2/allocate.c @@ -0,0 +1,73 @@ +/// @file + +#include "pkg/noun/allocate.h" +#include "pkg/noun/v1/allocate.h" +#include "pkg/noun/v2/allocate.h" + +#include "pkg/noun/v2/hashtable.h" +#include "log.h" +#include "pkg/noun/v2/manage.h" +#include "options.h" +#include "retrieve.h" +#include "trace.h" +#include "vortex.h" + +u3a_v2_road* u3a_v2_Road; + +u3_noun +u3a_v2_rewritten_noun(u3_noun som) +{ + if ( c3y == u3a_v2_is_cat(som) ) { + return som; + } + u3_post som_p = u3a_v2_rewritten(u3a_v1_to_off(som)); + + if ( c3y == u3a_v2_is_pug(som) ) { + som_p = u3a_v2_to_pug(som_p); + } + else { + som_p = u3a_v2_to_pom(som_p); + } + + return som_p; +} + +/* u3a_v2_rewrite_compact(): rewrite pointers in ad-hoc persistent road structures. +*/ +void +u3a_v2_rewrite_compact(void) +{ + u3a_v2_rewrite_noun(u3R_v2->ski.gul); + u3a_v2_rewrite_noun(u3R_v2->bug.tax); + u3a_v2_rewrite_noun(u3R_v2->bug.mer); + u3a_v2_rewrite_noun(u3R_v2->pro.don); + u3a_v2_rewrite_noun(u3R_v2->pro.day); + u3a_v2_rewrite_noun(u3R_v2->pro.trace); + u3h_v2_rewrite(u3R_v2->cax.har_p); + + u3R_v2->ski.gul = u3a_v2_rewritten_noun(u3R_v2->ski.gul); + u3R_v2->bug.tax = u3a_v2_rewritten_noun(u3R_v2->bug.tax); + u3R_v2->bug.mer = u3a_v2_rewritten_noun(u3R_v2->bug.mer); + u3R_v2->pro.don = u3a_v2_rewritten_noun(u3R_v2->pro.don); + u3R_v2->pro.day = u3a_v2_rewritten_noun(u3R_v2->pro.day); + u3R_v2->pro.trace = u3a_v2_rewritten_noun(u3R_v2->pro.trace); + u3R_v2->cax.har_p = u3a_v2_rewritten(u3R_v2->cax.har_p); +} + +void +u3a_v2_rewrite_noun(u3_noun som) +{ + if ( c3n == u3a_v2_is_cell(som) ) { + return; + } + + if ( c3n == u3a_v2_rewrite_ptr(u3a_v1_to_ptr((som))) ) return; + + u3a_v2_cell* cel = (u3a_v2_cell*) u3a_v1_to_ptr(som); + + u3a_v2_rewrite_noun(cel->hed); + u3a_v2_rewrite_noun(cel->tel); + + cel->hed = u3a_v2_rewritten_noun(cel->hed); + cel->tel = u3a_v2_rewritten_noun(cel->tel); +} diff --git a/pkg/noun/v2/allocate.h b/pkg/noun/v2/allocate.h new file mode 100644 index 0000000000..82b1826adb --- /dev/null +++ b/pkg/noun/v2/allocate.h @@ -0,0 +1,133 @@ +#ifndef U3_ALLOCATE_V2_H +#define U3_ALLOCATE_V2_H + +#include "pkg/noun/allocate.h" + +#include "pkg/noun/v2/manage.h" +#include "options.h" + + /** Aliases. + **/ +# define u3a_v2_botox u3a_botox +# define u3a_v2_box u3a_box +# define u3a_v2_cell u3a_cell +# define u3a_v2_fbox u3a_fbox +# define u3a_v2_fbox_no u3a_fbox_no +# define u3a_v2_free u3a_free +# define u3a_v2_heap u3a_heap +# define u3a_v2_into u3a_into +# define u3a_v2_is_cat u3a_is_cat +# define u3a_v2_is_cell u3a_is_cell +# define u3a_v2_is_north u3a_is_north +# define u3a_v2_is_pom u3a_is_pom +# define u3a_v2_is_pug u3a_is_pug +# define u3a_v2_malloc u3a_malloc +# define u3a_v2_minimum u3a_minimum +# define u3a_v2_outa u3a_outa +# define u3a_v2_pack_seek u3a_pack_seek +# define u3a_v2_rewrite u3a_rewrite +# define u3a_v2_rewrite_ptr u3a_rewrite_ptr +# define u3a_v2_rewritten u3a_rewritten +# define u3a_v2_to_off u3a_to_off +# define u3a_v2_to_ptr u3a_to_ptr +# define u3a_v2_to_wtr u3a_to_wtr +# define u3a_v2_to_pug u3a_to_pug +# define u3a_v2_to_pom u3a_to_pom +# define u3a_v2_wfree u3a_wfree + + /** Data structures. + **/ + /* u3a_v2_road: contiguous allocation and execution context. + */ + typedef struct _u3a_v2_road { + u3p(struct _u3a_v2_road) par_p; // parent road + u3p(struct _u3a_v2_road) kid_p; // child road list + u3p(struct _u3a_v2_road) nex_p; // sibling road + + u3p(c3_w) cap_p; // top of transient region + u3p(c3_w) hat_p; // top of durable region + u3p(c3_w) mat_p; // bottom of transient region + u3p(c3_w) rut_p; // bottom of durable region + u3p(c3_w) ear_p; // original cap if kid is live + + c3_w fut_w[32]; // futureproof buffer + + struct { // escape buffer + union { + jmp_buf buf; + c3_w buf_w[256]; // futureproofing + }; + } esc; + + struct { // miscellaneous config + c3_w fag_w; // flag bits + } how; // + + struct { // allocation pools + u3p(u3a_fbox) fre_p[u3a_fbox_no]; // heap by node size log + u3p(u3a_fbox) cel_p; // custom cell allocator + c3_w fre_w; // number of free words + c3_w max_w; // maximum allocated + } all; + + u3a_jets jed; // jet dashboard + + struct { // bytecode state + u3p(u3h_root) har_p; // formula->post of bytecode + } byc; + + struct { // namespace + u3_noun gul; // (list $+(* (unit (unit)))) now + } ski; + + struct { // trace stack + u3_noun tax; // (list ,*) + u3_noun mer; // emergency buffer to release + } bug; + + struct { // profile stack + c3_d nox_d; // nock steps + c3_d cel_d; // cell allocations + u3_noun don; // (list batt) + u3_noun trace; // (list trace) + u3_noun day; // doss, only in u3H (moveme) + } pro; + + struct { // memoization + u3p(u3h_root) har_p; // (map (pair term noun) noun) + } cax; + } u3a_v2_road; + + /** Globals. + **/ + /// Current road (thread-local). + extern u3a_v2_road* u3a_v2_Road; +# define u3R_v2 u3a_v2_Road + + /** Functions. + **/ + /** Allocation. + **/ + /* Reference and arena control. + */ + /* u3a_v2_rewrite_compact(): rewrite pointers in ad-hoc persistent road structures. + */ + void + u3a_v2_rewrite_compact(void); + + /* u3a_v2_rewrite_noun(): rewrite a noun for compaction. + */ + void + u3a_v2_rewrite_noun(u3_noun som); + + /* u3a_v2_rewritten(): rewrite a pointer for compaction. + */ + u3_post + u3a_v2_rewritten(u3_post som_p); + + /* u3a_v2_rewritten(): rewritten noun pointer for compaction. + */ + u3_noun + u3a_v2_rewritten_noun(u3_noun som); + +#endif /* ifndef U3_ALLOCATE_V2_H */ diff --git a/pkg/noun/v2/hashtable.c b/pkg/noun/v2/hashtable.c new file mode 100644 index 0000000000..6963d07fde --- /dev/null +++ b/pkg/noun/v2/hashtable.c @@ -0,0 +1,143 @@ +/// @file + +#include "pkg/noun/hashtable.h" +#include "pkg/noun/v1/hashtable.h" +#include "pkg/noun/v2/hashtable.h" + +#include "pkg/noun/allocate.h" +#include "pkg/noun/v1/allocate.h" +#include "pkg/noun/v2/allocate.h" + +/* _ch_v2_popcount(): number of bits set in word. A standard intrinsic. +** NB: copy of _ch_v2_popcount in pkg/noun/hashtable.c +*/ +static c3_w +_ch_v2_popcount(c3_w num_w) +{ + return __builtin_popcount(num_w); +} + +/* _ch_v2_free_buck(): free bucket +** NB: copy of _ch_v2_free_buck in pkg/noun/hashtable.c +*/ +static void +_ch_v2_free_buck(u3h_v2_buck* hab_u) +{ + c3_w i_w; + + for ( i_w = 0; i_w < hab_u->len_w; i_w++ ) { + u3z(u3h_v2_slot_to_noun(hab_u->sot_w[i_w])); + } + u3a_v2_wfree(hab_u); +} + +/* _ch_v2_free_node(): free node. +*/ +static void +_ch_v2_free_node(u3h_v2_node* han_u, c3_w lef_w) +{ + c3_w len_w = _ch_v2_popcount(han_u->map_w); + c3_w i_w; + + lef_w -= 5; + + for ( i_w = 0; i_w < len_w; i_w++ ) { + c3_w sot_w = han_u->sot_w[i_w]; + + if ( _(u3h_v2_slot_is_noun(sot_w)) ) { + u3z(u3h_v2_slot_to_noun(sot_w)); + } + else { + // NB: u3h_v2_slot_to_node() + void* hav_v = u3h_v2_slot_to_node(sot_w); + + if ( 0 == lef_w ) { + _ch_v2_free_buck(hav_v); + } else { + _ch_v2_free_node(hav_v, lef_w); + } + } + } + u3a_v2_wfree(han_u); +} + +/* _ch_v2_rewrite_buck(): rewrite buck for compaction. +*/ +void +_ch_v2_rewrite_buck(u3h_v2_buck* hab_u) +{ + if ( c3n == u3a_v2_rewrite_ptr(hab_u) ) return; + c3_w i_w; + + for ( i_w = 0; i_w < hab_u->len_w; i_w++ ) { + u3_noun som = u3h_v2_slot_to_noun(hab_u->sot_w[i_w]); + hab_u->sot_w[i_w] = u3h_v2_noun_to_slot(u3a_v2_rewritten_noun(som)); + u3a_v2_rewrite_noun(som); + } +} + +/* _ch_v2_rewrite_node(): rewrite node for compaction. +*/ +void +_ch_v2_rewrite_node(u3h_v2_node* han_u, c3_w lef_w) +{ + if ( c3n == u3a_v2_rewrite_ptr(han_u) ) return; + + c3_w len_w = _ch_v2_popcount(han_u->map_w); + c3_w i_w; + + lef_w -= 5; + + for ( i_w = 0; i_w < len_w; i_w++ ) { + c3_w sot_w = han_u->sot_w[i_w]; + + if ( _(u3h_v2_slot_is_noun(sot_w)) ) { + u3_noun kev = u3h_v2_slot_to_noun(sot_w); + han_u->sot_w[i_w] = u3h_v2_noun_to_slot(u3a_v2_rewritten_noun(kev)); + + u3a_v2_rewrite_noun(kev); + } + else { + void* hav_v = u3h_v1_slot_to_node(sot_w); + u3h_v2_node* nod_u = u3to(u3h_v2_node, u3a_v2_rewritten(u3of(u3h_v2_node,hav_v))); + + han_u->sot_w[i_w] = u3h_v2_node_to_slot(nod_u); + + if ( 0 == lef_w ) { + _ch_v2_rewrite_buck(hav_v); + } else { + _ch_v2_rewrite_node(hav_v, lef_w); + } + } + } +} + +/* u3h_v2_rewrite(): rewrite pointers during compaction. +*/ +void +u3h_v2_rewrite(u3p(u3h_v2_root) har_p) +{ + u3h_v2_root* har_u = u3to(u3h_v2_root, har_p); + c3_w i_w; + + if ( c3n == u3a_v2_rewrite_ptr(har_u) ) return; + + for ( i_w = 0; i_w < 64; i_w++ ) { + c3_w sot_w = har_u->sot_w[i_w]; + + if ( _(u3h_v2_slot_is_noun(sot_w)) ) { + u3_noun kev = u3h_v2_slot_to_noun(sot_w); + har_u->sot_w[i_w] = u3h_v2_noun_to_slot(u3a_v2_rewritten_noun(kev)); + + u3a_v2_rewrite_noun(kev); + } + else if ( _(u3h_v2_slot_is_node(sot_w)) ) { + u3h_v2_node* han_u = (u3h_v2_node*) u3h_v1_slot_to_node(sot_w); + u3h_v2_node* nod_u = u3to(u3h_v2_node, u3a_v2_rewritten(u3of(u3h_v2_node,han_u))); + + har_u->sot_w[i_w] = u3h_v2_node_to_slot(nod_u); + + _ch_v2_rewrite_node(han_u, 25); + } + } +} diff --git a/pkg/noun/v2/hashtable.h b/pkg/noun/v2/hashtable.h new file mode 100644 index 0000000000..be6ab9587c --- /dev/null +++ b/pkg/noun/v2/hashtable.h @@ -0,0 +1,41 @@ +#ifndef U3_HASHTABLE_V2_H +#define U3_HASHTABLE_V2_H + +#define u3h_v2_buck u3h_buck +#define u3h_v2_free u3h_free +#define u3h_v2_node u3h_node +#define u3h_v2_noun_to_slot u3h_noun_to_slot +#define u3h_v2_root u3h_root +#define u3h_v2_slot_is_node u3h_slot_is_node +#define u3h_v2_slot_is_noun u3h_slot_is_noun +#define u3h_v2_slot_to_noun u3h_slot_to_noun +#define u3h_v2_walk u3h_walk +#define u3h_v2_walk_with u3h_walk_with + +#include "pkg/noun/hashtable.h" + +#include "c3.h" +#include "types.h" + + /** Data structures. + **/ + + /** HAMT macros. + *** + *** Coordinate with u3_noun definition! + **/ + /* u3h_v2_slot_to_node(): slot to node pointer + ** u3h_v2_node_to_slot(): node pointer to slot + */ +# define u3h_v2_slot_to_node(sot) (u3a_v2_into(((sot) & 0x3fffffff) << u3a_vits)) +# define u3h_v2_node_to_slot(ptr) ((u3a_v2_outa((ptr)) >> u3a_vits) | 0x40000000) + + /** Functions. + *** + **/ + /* u3h_v2_rewrite(): rewrite hashtable for compaction. + */ + void + u3h_v2_rewrite(u3p(u3h_root) har_p); + +#endif /* U3_HASHTABLE_V2_H */ diff --git a/pkg/noun/v2/jets.c b/pkg/noun/v2/jets.c new file mode 100644 index 0000000000..e70ef46abe --- /dev/null +++ b/pkg/noun/v2/jets.c @@ -0,0 +1,37 @@ +/// @file + +#include "pkg/noun/vortex.h" + +#include "pkg/noun/jets.h" +#include "pkg/noun/v2/jets.h" + +#include "pkg/noun/v2/allocate.h" +#include "pkg/noun/v2/hashtable.h" +#include "pkg/noun/v2/vortex.h" + +/* u3j_v2_rewrite_compact(): rewrite jet state for compaction. + * + * NB: u3R_v2->jed.han_p *must* be cleared (currently via u3j_v2_reclaim above) + * since it contains hanks which are not nouns but have loom pointers. + * Alternately, rewrite the entries with u3h_v2_walk, using u3j_v2_mark as a + * template for how to walk. There's an untested attempt at this in git + * history at e8a307a. +*/ +void +u3j_v2_rewrite_compact() +{ + u3h_v2_rewrite(u3R_v2->jed.war_p); + u3h_v2_rewrite(u3R_v2->jed.cod_p); + u3h_v2_rewrite(u3R_v2->jed.han_p); + u3h_v2_rewrite(u3R_v2->jed.bas_p); + + if ( u3R_v2 == &(u3H_v2->rod_u) ) { + u3h_v2_rewrite(u3R_v2->jed.hot_p); + u3R_v2->jed.hot_p = u3a_v2_rewritten(u3R_v2->jed.hot_p); + } + + u3R_v2->jed.war_p = u3a_v2_rewritten(u3R_v2->jed.war_p); + u3R_v2->jed.cod_p = u3a_v2_rewritten(u3R_v2->jed.cod_p); + u3R_v2->jed.han_p = u3a_v2_rewritten(u3R_v2->jed.han_p); + u3R_v2->jed.bas_p = u3a_v2_rewritten(u3R_v2->jed.bas_p); +} diff --git a/pkg/noun/v2/jets.h b/pkg/noun/v2/jets.h new file mode 100644 index 0000000000..a596170f60 --- /dev/null +++ b/pkg/noun/v2/jets.h @@ -0,0 +1,22 @@ +/// @file + +#ifndef U3_JETS_V2_H +#define U3_JETS_V2_H + + /** Aliases. + **/ +# define u3j_v2_fink u3j_fink +# define u3j_v2_fist u3j_fist +# define u3j_v2_rite u3j_rite +# define u3j_v2_rite_lose u3j_rite_lose +# define u3j_v2_site u3j_site +# define u3j_v2_site_lose u3j_site_lose + + /** Functions. + **/ + /* u3j_v2_rewrite_compact(): rewrite jet state for compaction. + */ + void + u3j_v2_rewrite_compact(); + +#endif /* ifndef U3_JETS_V2_H */ diff --git a/pkg/noun/v2/manage.c b/pkg/noun/v2/manage.c new file mode 100644 index 0000000000..525f356944 --- /dev/null +++ b/pkg/noun/v2/manage.c @@ -0,0 +1,177 @@ +/// @file + +#include "pkg/noun/v2/manage.h" + +#include "pkg/noun/v2/allocate.h" +#include "pkg/noun/v2/hashtable.h" +#include "pkg/noun/v2/jets.h" +#include "pkg/noun/v2/nock.h" +#include "pkg/noun/v2/options.h" +#include "pkg/noun/vortex.h" +#include "pkg/noun/v2/vortex.h" + +/* _cm_pack_rewrite(): trace through arena, rewriting pointers. + * XX need to version; dynamic scope insanity! +*/ +static void +_cm_pack_rewrite(void) +{ + // XX fix u3a_v2_rewrite* to support south roads + // + u3_assert( &(u3H_v2->rod_u) == u3R_v2 ); + + u3v_v2_rewrite_compact(); + u3j_v2_rewrite_compact(); + u3n_v2_rewrite_compact(); + u3a_v2_rewrite_compact(); +} + +static void +_migrate_reclaim() +{ + // XX update this and similar printfs + fprintf(stderr, "loom: migration reclaim\r\n"); + u3m_v1_reclaim(); +} + +static void +_migrate_seek(const u3a_v2_road *rod_u) +{ + /* + very much like u3a_v2_pack_seek with the following changes: + - there is no need to account for free space as |pack is performed before + the migration + - odd sized boxes will be padded by one word to achieve an even size + - rut will be moved from one word ahead of u3_Loom to two words ahead + */ + c3_w * box_w = u3a_v2_into(rod_u->rut_p); + c3_w * end_w = u3a_v2_into(rod_u->hat_p); + u3_post new_p = (rod_u->rut_p + 1 + c3_wiseof(u3a_v2_box)); + u3a_v2_box * box_u = (void *)box_w; + + fprintf(stderr, "loom: migration seek\r\n"); + + for (; box_w < end_w + ; box_w += box_u->siz_w + , box_u = (void*)box_w) + { + if (!box_u->use_w) + continue; + u3_assert(box_u->siz_w); + u3_assert(box_u->use_w); + box_w[box_u->siz_w - 1] = new_p; + new_p = c3_align(new_p + box_u->siz_w, 2, C3_ALGHI); + } +} + +static void +_migrate_rewrite() +{ + fprintf(stderr, "loom: migration rewrite\r\n"); + + _cm_pack_rewrite(); +} + +static void +_migrate_move(u3a_v2_road *rod_u) +{ + fprintf(stderr, "loom: migration move\r\n"); + + c3_z hiz_z = u3a_v2_heap(rod_u) * sizeof(c3_w); + + /* calculate required shift distance to prevent write head overlapping read head */ + c3_w off_w = 1; /* at least 1 word because u3R_v2->rut_p migrates from 1 to 2 */ + for (u3a_v2_box *box_u = u3a_v2_into(rod_u->rut_p) + ; (void *)box_u < u3a_v2_into(rod_u->hat_p) + ; box_u = (void *)((c3_w *)box_u + box_u->siz_w)) + off_w += box_u->siz_w & 1; /* odd-sized boxes are padded by one word */ + + /* shift */ + memmove(u3a_v2_into(u3H_v2->rod_u.rut_p + off_w), + u3a_v2_into(u3H_v2->rod_u.rut_p), + hiz_z); + /* manually zero the former rut */ + *(c3_w *)u3a_v2_into(rod_u->rut_p) = 0; + + /* relocate boxes to DWORD-aligned addresses stored in trailing size word */ + c3_w *box_w = u3a_v2_into(rod_u->rut_p + off_w); + c3_w *end_w = u3a_v2_into(rod_u->hat_p + off_w); + u3a_v2_box *old_u = (void *)box_w; + c3_w siz_w = old_u->siz_w; + u3p(c3_w) new_p = rod_u->rut_p + 1 + c3_wiseof(u3a_v2_box); + c3_w *new_w; + + for (; box_w < end_w + ; box_w += siz_w + , old_u = (void *)box_w + , siz_w = old_u->siz_w) { + old_u->use_w &= 0x7fffffff; + + if (!old_u->use_w) + continue; + + new_w = (void *)u3a_v2_botox(u3a_v2_into(new_p)); + u3_assert(box_w[siz_w - 1] == new_p); + u3_assert(new_w <= box_w); + + c3_w i_w; + for (i_w = 0; i_w < siz_w - 1; i_w++) + new_w[i_w] = box_w[i_w]; + + if (siz_w & 1) { + new_w[i_w++] = 0; /* pad odd sized boxes */ + new_w[i_w++] = siz_w + 1; /* restore trailing size word */ + new_w[0] = siz_w + 1; /* and the leading size word */ + } + else { + new_w[i_w++] = siz_w; + } + + new_p += i_w; + } + + /* restore proper heap state */ + rod_u->rut_p = 2; + rod_u->hat_p = new_p - c3_wiseof(u3a_v2_box); + + /* like |pack, clear the free lists and cell allocator */ + for (c3_w i_w = 0; i_w < u3a_v2_fbox_no; i_w++) + u3R_v2->all.fre_p[i_w] = 0; + + u3R_v2->all.fre_w = 0; + u3R_v2->all.cel_p = 0; +} + + +/* u3m_v2_migrate: perform loom migration if necessary. +*/ +void +u3m_v2_migrate() +{ + c3_w len_w = u3C_v2.wor_i - 1; + c3_w ver_w = *(u3_Loom + len_w); + + u3_assert( U3V_VER1 == ver_w ); + + c3_w* mem_w = u3_Loom + 1; + c3_w siz_w = c3_wiseof(u3v_v2_home); + c3_w* mat_w = (mem_w + len_w) - siz_w; + + u3H_v2 = (void *)mat_w; + u3R_v2 = &u3H_v2->rod_u; + + u3R_v2->cap_p = u3R_v2->mat_p = u3a_v2_outa(u3H_v2); + + fprintf(stderr, "loom: pointer compression migration running...\r\n"); + + /* perform the migration in a pattern similar to |pack */ + _migrate_reclaim(); + _migrate_seek(&u3H_v2->rod_u); + _migrate_rewrite(); + _migrate_move(&u3H_v2->rod_u); + + /* finally update the version and commit to disk */ + u3H_v2->ver_w = U3V_VER2; + + fprintf(stderr, "loom: pointer compression migration done\r\n"); +} diff --git a/pkg/noun/v2/manage.h b/pkg/noun/v2/manage.h new file mode 100644 index 0000000000..998a37cc65 --- /dev/null +++ b/pkg/noun/v2/manage.h @@ -0,0 +1,14 @@ +/// @file + +#ifndef U3_MANAGE_V2_H +#define U3_MANAGE_V2_H + + /** System management. + **/ + /* u3m_v2_migrate: perform pointer compression loom migration if necessary. + ver_w - target version + */ + void + u3m_v2_migrate(); + +#endif /* ifndef U3_MANAGE_V2_H */ diff --git a/pkg/noun/v2/nock.c b/pkg/noun/v2/nock.c new file mode 100644 index 0000000000..95a011e005 --- /dev/null +++ b/pkg/noun/v2/nock.c @@ -0,0 +1,28 @@ +/// @file + +#include "pkg/noun/nock.h" +#include "pkg/noun/v2/nock.h" + +#include "pkg/noun/v2/allocate.h" +#include "pkg/noun/v2/hashtable.h" +#include "pkg/noun/v2/vortex.h" + +/* u3n_v2_rewrite_compact(): rewrite the bytecode cache for compaction. + * + * NB: u3R_v2->byc.har_p *must* be cleared (currently via u3n_v2_reclaim above), + * since it contains things that look like nouns but aren't. + * Specifically, it contains "cells" where the tail is a + * pointer to a u3a_v2_malloc'ed block that contains loom pointers. + * + * You should be able to walk this with u3h_v2_walk and rewrite the + * pointers, but you need to be careful to handle that u3a_v2_malloc + * pointers can't be turned into a box by stepping back two words. You + * must step back one word to get the padding, step then step back that + * many more words (plus one?). + */ +void +u3n_v2_rewrite_compact() +{ + u3h_v2_rewrite(u3R_v2->byc.har_p); + u3R_v2->byc.har_p = u3a_v2_rewritten(u3R_v2->byc.har_p); +} diff --git a/pkg/noun/v2/nock.h b/pkg/noun/v2/nock.h new file mode 100644 index 0000000000..eddc3d2074 --- /dev/null +++ b/pkg/noun/v2/nock.h @@ -0,0 +1,18 @@ +/// @file + +#ifndef U3_NOCK_V2_H +#define U3_NOCK_V2_H + + /** Aliases. + **/ +#define u3n_v2_free u3n_free +#define u3n_v2_prog u3n_prog + + /** Functions. + **/ + /* u3n_v2_rewrite_compact(): rewrite bytecode cache for compaction. + */ + void + u3n_v2_rewrite_compact(); + +#endif /* ifndef U3_NOCK_V2_H */ diff --git a/pkg/noun/v2/options.h b/pkg/noun/v2/options.h new file mode 100644 index 0000000000..4af573f20d --- /dev/null +++ b/pkg/noun/v2/options.h @@ -0,0 +1,14 @@ +/// @file + +#ifndef U3_OPTIONS_V2_H +#define U3_OPTIONS_V2_H + +#include "pkg/noun/options.h" + + /** Globals. + **/ + /* u3_Config / u3C: global memory control. + */ +# define u3C_v2 u3C + +#endif /* ifndef U3_OPTIONS_H */ diff --git a/pkg/noun/v2/vortex.c b/pkg/noun/v2/vortex.c new file mode 100644 index 0000000000..382ed24192 --- /dev/null +++ b/pkg/noun/v2/vortex.c @@ -0,0 +1,25 @@ +/// @file + +#include "pkg/noun/vortex.h" +#include "pkg/noun/v2/vortex.h" + +#include "pkg/noun/v2/allocate.h" + +u3v_v2_home* u3v_v2_Home; + +/* u3v_v2_rewrite_compact(): rewrite arvo kernel for compaction. +*/ +void +u3v_v2_rewrite_compact() +{ + u3v_v2_arvo* arv_u = &(u3H_v2->arv_u); + + u3a_v2_rewrite_noun(arv_u->roc); + u3a_v2_rewrite_noun(arv_u->now); + u3a_v2_rewrite_noun(arv_u->yot); + + arv_u->roc = u3a_v2_rewritten_noun(arv_u->roc); + arv_u->now = u3a_v2_rewritten_noun(arv_u->now); + arv_u->yot = u3a_v2_rewritten_noun(arv_u->yot); +} + diff --git a/pkg/noun/v2/vortex.h b/pkg/noun/v2/vortex.h new file mode 100644 index 0000000000..9ef060e610 --- /dev/null +++ b/pkg/noun/v2/vortex.h @@ -0,0 +1,40 @@ +/// @file + +#ifndef U3_VORTEX_V2_H +#define U3_VORTEX_V2_H + +#include "pkg/noun/vortex.h" + +#include "pkg/noun/v2/allocate.h" +#include "pkg/noun/version.h" + + /** Aliases. + **/ +# define u3v_v2_arvo u3v_arvo + + /** Data structures. + **/ + /* u3v_v2_home: all internal (within image) state. + ** NB: version must be last for discriminability in north road + */ + typedef struct _u3v_v2_home { + u3a_v2_road rod_u; // storage state + u3v_v2_arvo arv_u; // arvo state + u3v_version ver_w; // version number + } u3v_v2_home; + + /** Globals. + **/ + /// Arvo internal state. + extern u3v_v2_home* u3v_v2_Home; +# define u3H_v2 u3v_v2_Home +# define u3A_v2 (&(u3v_v2_Home->arv_u)) + + /** Functions. + **/ + /* u3v_v2_rewrite_compact(): rewrite arvo kernel for compaction. + */ + void + u3v_v2_rewrite_compact(); + +#endif /* ifndef U3_VORTEX_V2_H */ diff --git a/pkg/noun/vortex.h b/pkg/noun/vortex.h index cbd0f295ff..2d202e831c 100644 --- a/pkg/noun/vortex.h +++ b/pkg/noun/vortex.h @@ -37,9 +37,6 @@ # define u3H u3v_Home # define u3A (&(u3v_Home->arv_u)) - /** Constants. - **/ - /** Functions. **/ /* u3v_life(): execute initial lifecycle, producing Arvo core. From f282a167ad154dd2c279963282920e800fc23b61 Mon Sep 17 00:00:00 2001 From: Matthew LeVan Date: Tue, 1 Aug 2023 12:09:44 -0400 Subject: [PATCH 091/271] vere: bump %lull kelvin to %322 --- pkg/vere/pier.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/vere/pier.c b/pkg/vere/pier.c index 1e39d14ef4..506a59a195 100644 --- a/pkg/vere/pier.c +++ b/pkg/vere/pier.c @@ -676,7 +676,7 @@ _pier_wyrd_fail(u3_pier* pir_u, u3_ovum* egg_u, u3_noun lud) // #define VERE_NAME "vere" #define VERE_ZUSE 412 -#define VERE_LULL 323 +#define VERE_LULL 322 /* _pier_wyrd_aver(): check for %wend effect and version downgrade. RETAIN */ From f18ec083824ca10df51679ea9d92595da5673c94 Mon Sep 17 00:00:00 2001 From: Matthew LeVan Date: Tue, 1 Aug 2023 12:10:18 -0400 Subject: [PATCH 092/271] vere: decrement zuse to %411 --- pkg/vere/pier.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/vere/pier.c b/pkg/vere/pier.c index 506a59a195..8e54dbb37e 100644 --- a/pkg/vere/pier.c +++ b/pkg/vere/pier.c @@ -675,7 +675,7 @@ _pier_wyrd_fail(u3_pier* pir_u, u3_ovum* egg_u, u3_noun lud) // XX organizing version constants // #define VERE_NAME "vere" -#define VERE_ZUSE 412 +#define VERE_ZUSE 411 #define VERE_LULL 322 /* _pier_wyrd_aver(): check for %wend effect and version downgrade. RETAIN From 0335f048f857c5064c35237b552561bac932c290 Mon Sep 17 00:00:00 2001 From: Sidnym Ladrut Date: Sun, 17 Sep 2023 19:39:13 +0000 Subject: [PATCH 093/271] add fixes proposed by ~finmep-lanteb --- pkg/noun/jets/tree.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/noun/jets/tree.c b/pkg/noun/jets/tree.c index e2ce880293..03c7b64e48 100644 --- a/pkg/noun/jets/tree.c +++ b/pkg/noun/jets/tree.c @@ -2139,7 +2139,6 @@ static c3_c* _k140_ha[] = { /* new jets */ - static u3j_harm _139_two_mate_a[] = {{".2", u3wb_mate, c3y}, {}}; static u3j_harm _139_hex_json_de_a[] = {{".2", u3we_json_de}, {}}; static u3j_harm _139_hex_json_en_a[] = {{".2", u3we_json_en}, {}}; static u3j_core _139_hex_json_d[] = @@ -2273,6 +2272,8 @@ static u3j_core _139_two__in_d[] = {} }; +static u3j_harm _139_two_mate_a[] = {{".2", u3wb_mate, c3y}, {}}; + static u3j_core _139_two_d[] = { { "tri", 3, 0, _139_tri_d, no_hashes, _140_tri_ho }, From 28f18498129173196158c18463e9320b71f2d36c Mon Sep 17 00:00:00 2001 From: Pyry Kovanen Date: Mon, 18 Sep 2023 17:00:34 +0300 Subject: [PATCH 094/271] Update VERSION to 3.0 --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 3e162f02ea..9f55b2ccb5 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.12 +3.0 From ad238b55529f5178f45cd459fbfe2bfa2095936b Mon Sep 17 00:00:00 2001 From: Ted Blackman Date: Tue, 19 Sep 2023 18:27:50 -0400 Subject: [PATCH 095/271] stun: use later RFC for packet format --- pkg/vere/io/ames.c | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/pkg/vere/io/ames.c b/pkg/vere/io/ames.c index ba0774e1f1..e66762c127 100644 --- a/pkg/vere/io/ames.c +++ b/pkg/vere/io/ames.c @@ -1289,9 +1289,21 @@ _stun_send_request(u3_ames* sam_u) add_u.sin_addr.s_addr = htonl(sam_u->sun_u.lan_u.pip_w); add_u.sin_port = htons(sam_u->sun_u.lan_u.por_s); + // see STUN RFC 8489 + // https://datatracker.ietf.org/doc/html/rfc8489#section-5 c3_y buf_y[20] = {0}; - buf_y[1] = 0x1; // message type "binding request" - memcpy(buf_y + 4, sam_u->sun_u.tid_y, 16); // TODO convert byte order? + + // STUN message type: "binding request" + buf_y[1] = 0x01; + + // STUN message length: 0 (header-only) + // (implicit in initialization) + + // STUN "magic cookie" 0x2112A442 in network byte order + buf_y[4] = 0x21; buf_y[5] = 0x12; buf_y[6] = 0xa4; buf_y[7] = 0x42; + + // STUN "transaction id" + memcpy(buf_y + 8, sam_u->sun_u.tid_y, 12); uv_buf_t buf_u = uv_buf_init((c3_c*)buf_y, 20); u3_stun_send* snd_u = c3_calloc(sizeof(*snd_u)); From d5dd64807c85f9d2f0a7cd217acf21b7f2aa6a26 Mon Sep 17 00:00:00 2001 From: Joe Bryan Date: Fri, 22 Sep 2023 14:48:13 -0400 Subject: [PATCH 096/271] vere: support stack traces on failing boot / full replay --- pkg/noun/vortex.c | 7 +------ pkg/vere/mars.c | 25 +++++++++++++++++++++++++ pkg/vere/serf.c | 25 ++++++++++++++++++++++--- 3 files changed, 48 insertions(+), 9 deletions(-) diff --git a/pkg/noun/vortex.c b/pkg/noun/vortex.c index 5981068b87..0bb9af8c29 100644 --- a/pkg/noun/vortex.c +++ b/pkg/noun/vortex.c @@ -39,18 +39,12 @@ c3_o u3v_boot(u3_noun eve) { c3_d len_d; - { u3_noun len = u3qb_lent(eve); u3_assert( c3y == u3r_safe_chub(len, &len_d) ); u3z(len); } - // ensure zero-initialized kernel - // - u3A->roc = 0; - u3A->eve_d = 0; - { u3_noun pro = u3m_soft(0, u3v_life, eve); @@ -59,6 +53,7 @@ u3v_boot(u3_noun eve) return c3n; } + u3z(u3A->roc); u3A->roc = u3k(u3t(pro)); u3A->eve_d = len_d; u3z(pro); diff --git a/pkg/vere/mars.c b/pkg/vere/mars.c index 37933bd49e..4751391bd9 100644 --- a/pkg/vere/mars.c +++ b/pkg/vere/mars.c @@ -6,6 +6,8 @@ #include "noun.h" #include "types.h" #include "vere.h" +#include "ivory.h" +#include "ur.h" #include "db/lmdb.h" #include #include @@ -179,6 +181,29 @@ _mars_do_boot(u3_disk* log_u, c3_d eve_d) // eve = u3m_love(u3ke_cue(u3ke_jam(eve))); + // install an ivory pill to support stack traces + // + // XX support -J + // + { + c3_d len_d = u3_Ivory_pill_len; + c3_y* byt_y = u3_Ivory_pill; + u3_cue_xeno* sil_u = u3s_cue_xeno_init_with(ur_fib27, ur_fib28); + u3_weak pil; + + if ( u3_none == (pil = u3s_cue_xeno_with(sil_u, len_d, byt_y)) ) { + u3l_log("lite: unable to cue ivory pill"); + exit(1); + } + + u3s_cue_xeno_done(sil_u); + + if ( c3n == u3v_boot_lite(pil)) { + u3l_log("lite: boot failed"); + exit(1); + } + } + u3l_log("--------------- bootstrap starting ----------------"); u3l_log("boot: 1-%u", u3qb_lent(eve)); diff --git a/pkg/vere/serf.c b/pkg/vere/serf.c index 3a59005e00..cfbbc47be3 100644 --- a/pkg/vere/serf.c +++ b/pkg/vere/serf.c @@ -3,6 +3,8 @@ #include "noun.h" #include "vere.h" +#include "ivory.h" +#include "ur.h" /* |% @@ -538,11 +540,28 @@ _serf_play_life(u3_serf* sef_u, u3_noun eve) u3z(len); } - // ensure zero-initialized kernel + // install an ivory pill to support stack traces // - // XX assert? + // XX support -J // - u3A->roc = 0; + { + c3_d len_d = u3_Ivory_pill_len; + c3_y* byt_y = u3_Ivory_pill; + u3_cue_xeno* sil_u = u3s_cue_xeno_init_with(ur_fib27, ur_fib28); + u3_weak pil; + + if ( u3_none == (pil = u3s_cue_xeno_with(sil_u, len_d, byt_y)) ) { + u3l_log("lite: unable to cue ivory pill"); + exit(1); + } + + u3s_cue_xeno_done(sil_u); + + if ( c3n == u3v_boot_lite(pil)) { + u3l_log("lite: boot failed"); + exit(1); + } + } gon = u3m_soft(0, u3v_life, eve); From 849cdc3751e675fcf824df3f49328e85ab0953be Mon Sep 17 00:00:00 2001 From: Matthew LeVan Date: Tue, 26 Sep 2023 21:21:19 -0400 Subject: [PATCH 097/271] build: fetch `urcrypt` instead of vendoring it --- .github/workflows/shared.yml | 5 +- README.md | 2 - WORKSPACE.bazel | 50 +--- bazel/BUILD.bazel | 3 +- bazel/third_party/urcrypt/BUILD.bazel | 0 bazel/third_party/urcrypt/urcrypt.BUILD | 28 ++ pkg/noun/BUILD.bazel | 2 +- pkg/noun/jets.c | 2 +- pkg/noun/manage.c | 2 +- pkg/urcrypt/.gitignore | 56 ---- pkg/urcrypt/BUILD.bazel | 36 --- pkg/urcrypt/README.md | 33 --- pkg/urcrypt/aes_cbc.c | 181 ------------- pkg/urcrypt/aes_ecb.c | 111 -------- pkg/urcrypt/aes_siv.c | 181 ------------- pkg/urcrypt/argon.c | 120 --------- pkg/urcrypt/argon2/argon2-specs.pdf | Bin 459608 -> 0 bytes pkg/urcrypt/argon2/latex/pics/argon2-par.pdf | Bin 76165 -> 0 bytes pkg/urcrypt/argon2/latex/pics/compression.pdf | Bin 40828 -> 0 bytes pkg/urcrypt/argon2/latex/pics/generic.pdf | Bin 30736 -> 0 bytes .../argon2/latex/pics/power-distribution.jpg | Bin 25673 -> 0 bytes pkg/urcrypt/ed25519.c | 53 ---- pkg/urcrypt/ge-additions/BUILD.bazel | 11 - pkg/urcrypt/ge-additions/LICENSE | 27 -- pkg/urcrypt/ge-additions/README.md | 20 -- pkg/urcrypt/ge-additions/ge-additions.c | 169 ------------ pkg/urcrypt/ge-additions/ge-additions.h | 8 - pkg/urcrypt/ge_additions.c | 126 --------- pkg/urcrypt/keccak.c | 21 -- pkg/urcrypt/ripemd.c | 20 -- pkg/urcrypt/scrypt.c | 31 --- pkg/urcrypt/secp256k1.c | 245 ----------------- pkg/urcrypt/sha.c | 49 ---- pkg/urcrypt/urcrypt.h | 246 ------------------ pkg/urcrypt/util.c | 14 - pkg/urcrypt/util.h | 6 - 36 files changed, 44 insertions(+), 1814 deletions(-) create mode 100644 bazel/third_party/urcrypt/BUILD.bazel create mode 100644 bazel/third_party/urcrypt/urcrypt.BUILD delete mode 100644 pkg/urcrypt/.gitignore delete mode 100644 pkg/urcrypt/BUILD.bazel delete mode 100644 pkg/urcrypt/README.md delete mode 100644 pkg/urcrypt/aes_cbc.c delete mode 100644 pkg/urcrypt/aes_ecb.c delete mode 100644 pkg/urcrypt/aes_siv.c delete mode 100644 pkg/urcrypt/argon.c delete mode 100644 pkg/urcrypt/argon2/argon2-specs.pdf delete mode 100644 pkg/urcrypt/argon2/latex/pics/argon2-par.pdf delete mode 100644 pkg/urcrypt/argon2/latex/pics/compression.pdf delete mode 100644 pkg/urcrypt/argon2/latex/pics/generic.pdf delete mode 100644 pkg/urcrypt/argon2/latex/pics/power-distribution.jpg delete mode 100644 pkg/urcrypt/ed25519.c delete mode 100644 pkg/urcrypt/ge-additions/BUILD.bazel delete mode 100644 pkg/urcrypt/ge-additions/LICENSE delete mode 100644 pkg/urcrypt/ge-additions/README.md delete mode 100644 pkg/urcrypt/ge-additions/ge-additions.c delete mode 100644 pkg/urcrypt/ge-additions/ge-additions.h delete mode 100644 pkg/urcrypt/ge_additions.c delete mode 100644 pkg/urcrypt/keccak.c delete mode 100644 pkg/urcrypt/ripemd.c delete mode 100644 pkg/urcrypt/scrypt.c delete mode 100644 pkg/urcrypt/secp256k1.c delete mode 100644 pkg/urcrypt/sha.c delete mode 100644 pkg/urcrypt/urcrypt.h delete mode 100644 pkg/urcrypt/util.c delete mode 100644 pkg/urcrypt/util.h diff --git a/.github/workflows/shared.yml b/.github/workflows/shared.yml index eec7f3fec9..d2f4959d0c 100644 --- a/.github/workflows/shared.yml +++ b/.github/workflows/shared.yml @@ -43,7 +43,7 @@ jobs: # runner with BuildJet instead. - { target: linux-aarch64, runner: buildjet-2vcpu-ubuntu-2204-arm } - { target: linux-x86_64, runner: ubuntu-22.04 } - # GitHub doesn't provide macOS machines with Apple Silicon, so we + # GitHub doesn't provide macOS machines with Apple Silicon, so we # self-host a runner with MacStadium instead. - { target: macos-aarch64, runner: [self-hosted, macos, ARM64] } - { target: macos-x86_64, runner: macos-12 } @@ -72,7 +72,7 @@ jobs: /private/var/tmp/_bazel_$(whoami) # Cache musl libc toolchains. /usr/local/*-musl - + - name: chown /usr/local/*-musl if: ${{ matrix.target == 'linux-x86_64' || matrix.target == 'linux-aarch64'}} run: | @@ -119,6 +119,7 @@ jobs: bazel run //bazel/toolchain:x86_64-linux-musl-gcc ;; "macos-aarch64") + brew install pkg-config ;; "macos-x86_64") # Switch Xcode path to match the path specified in our bazel toolchain. diff --git a/README.md b/README.md index 9e68fafdad..679d829fac 100644 --- a/README.md +++ b/README.md @@ -19,8 +19,6 @@ defined in its own package: - [`pkg/c3`](pkg/c3): A set of basic utilities for writing Urbit's style of C. - [`pkg/ent`](pkg/ent): A cross-platform wrapper for `getentropy(2)`. -- [`pkg/urcrypt`](pkg/urcrypt): A standardized interface for calling various - cryptographic functions used in the Urbit runtime. - [`pkg/ur`](pkg/ur): An implementation of [jam][jam] and [cue][cue], Urbit's bitwise noun serialization and deserialization algorithms, respectively. - [`pkg/noun`](pkg/noun): The Nock virtual machine and snapshotting system. diff --git a/WORKSPACE.bazel b/WORKSPACE.bazel index 2cad948ace..c5a66ad095 100644 --- a/WORKSPACE.bazel +++ b/WORKSPACE.bazel @@ -121,15 +121,6 @@ versioned_http_archive( version = "9681279cfaa6e6399bb7ca3afbbc27fc2e19df4b", ) -versioned_http_archive( - name = "argon2", - build_file = "//bazel/third_party/argon2:argon2.BUILD", - sha256 = "40850e6e6324be10f14228d17b617ad2074bb926eeddd6fe40ad5df833833949", - strip_prefix = "argon2-{version}", - url = "https://github.com/urbit/argon2/archive/{version}.tar.gz", - version = "a4c1e3f7138c2e577376beb99f964cf71e1c8b1b", -) - versioned_http_archive( name = "bazel_gazelle", sha256 = "efbbba6ac1a4fd342d5122cbdfdb82aeb2cf2862e35022c752eaddffada7c3f3", @@ -154,15 +145,6 @@ versioned_http_archive( version = "7.85.0", ) -versioned_http_archive( - name = "ed25519", - build_file = "//bazel/third_party/ed25519:ed25519.BUILD", - sha256 = "373923c85f61276e3cad2c0ae7a5d5cd4809ffe46c5abc1dc8276683a55782a0", - strip_prefix = "ed25519-{version}", - url = "https://github.com/orlp/ed25519/archive/{version}.tar.gz", - version = "7fa6712ef5d581a6981ec2b08ee623314cd1d1c4", -) - versioned_http_archive( name = "gmp", build_file = "//bazel/third_party/gmp:gmp.BUILD", @@ -214,17 +196,6 @@ versioned_http_file( version = "255fb1ca8206072f1d09425f0db61ecfe7ff5b17", ) -versioned_http_archive( - name = "keccak_tiny", - build_file = "//bazel/third_party/keccak_tiny:keccak_tiny.BUILD", - patch_args = ["-p1"], - patches = ["//bazel/third_party/keccak_tiny:{version}.patch"], - sha256 = "6d4717f96b84805886c74bad89e911076664d992f197634fd7cdfca2ac0f62ef", - strip_prefix = "keccak-tiny-{version}", - url = "https://github.com/coruus/keccak-tiny/archive/{version}.tar.gz", - version = "64b6647514212b76ae7bca0dea9b7b197d1d8186", -) - versioned_http_archive( name = "lmdb", build_file = "//bazel/third_party/lmdb:lmdb.BUILD", @@ -272,18 +243,6 @@ versioned_http_archive( version = "67108d883061043e55d0fb13961ac1b6fc8a485c", ) -versioned_http_archive( - name = "scrypt", - build_file = "//bazel/third_party/scrypt:scrypt.BUILD", - sha256 = "df681fb19b653b1a12970ebb6091bb2b58411b9e7baf01143870f6be3f099541", - strip_prefix = "libscrypt-{version}", - url = "https://github.com/technion/libscrypt/archive/{version}.tar.gz", - # When bumping the version, compare `Makefile` in the `scrypt` repo to - # {build_file} and confirm that {build_file} remains an accurate description - # of the scrypt build process. - version = "60e585cdd752262b22ed4113eca41c0461a61608", -) - versioned_http_archive( name = "secp256k1", build_file = "//bazel/third_party/secp256k1:secp256k1.BUILD", @@ -337,6 +296,15 @@ versioned_http_file( version = "ea8fee3aa0434d4bdf1bf785e5ec346c7ecba7fd", ) +versioned_http_archive( + name = "urcrypt", + build_file = "//bazel/third_party/urcrypt:urcrypt.BUILD", + sha256 = "00ec597c14c418802d5db2d6a68cf83bd4f5419071b95f979374d3184599d6c8", + strip_prefix = "urcrypt-{version}", + url = "https://github.com/urbit/urcrypt/archive/{version}.tar.gz", + version = "b970baefa6e0a680fffa2b2ee19c956a4ae20355", +) + versioned_http_archive( name = "uv", build_file = "//bazel/third_party/uv:uv.BUILD", diff --git a/bazel/BUILD.bazel b/bazel/BUILD.bazel index bb93f5d26c..5ac74d0b96 100644 --- a/bazel/BUILD.bazel +++ b/bazel/BUILD.bazel @@ -11,7 +11,6 @@ refresh_compile_commands( "//pkg/ent", "//pkg/noun", "//pkg/ur", - "//pkg/urcrypt", "//pkg/vere:urbit", ], # No need to add flags already in .bazelrc. They're automatically picked up. @@ -19,4 +18,4 @@ refresh_compile_commands( # Wildcard patterns, like //... for everything, *are* allowed here, just like a build. # As are additional targets (+) and subtractions (-), like in bazel query https://docs.bazel.build/versions/main/query.html#expressions # And if you're working on a header-only library, specify a test or binary target that compiles it. -) \ No newline at end of file +) diff --git a/bazel/third_party/urcrypt/BUILD.bazel b/bazel/third_party/urcrypt/BUILD.bazel new file mode 100644 index 0000000000..e69de29bb2 diff --git a/bazel/third_party/urcrypt/urcrypt.BUILD b/bazel/third_party/urcrypt/urcrypt.BUILD new file mode 100644 index 0000000000..973830447d --- /dev/null +++ b/bazel/third_party/urcrypt/urcrypt.BUILD @@ -0,0 +1,28 @@ +load("@rules_foreign_cc//foreign_cc:defs.bzl", "configure_make") + +filegroup( + name = "all", + srcs = glob(["**"]), +) + +configure_make( + name = "urcrypt", + autogen = True, + configure_in_place = True, + configure_options = [ + "--disable-shared", + ], + copts = [ + "-Wall", + "-g", + "-O3", + ], + deps = [ + "@aes_siv", + "@openssl", + "@secp256k1" + ], + lib_source = ":all", + out_static_libs = ["liburcrypt.a"], + visibility = ["//visibility:public"], +) diff --git a/pkg/noun/BUILD.bazel b/pkg/noun/BUILD.bazel index 964e4bf60f..a4efaeaff6 100644 --- a/pkg/noun/BUILD.bazel +++ b/pkg/noun/BUILD.bazel @@ -34,13 +34,13 @@ vere_library( "//pkg/c3", "//pkg/ent", "//pkg/ur", - "//pkg/urcrypt", "@gmp", "@murmur3", "@openssl", "@pdjson", "@sigsegv", "@softfloat", + "@urcrypt", ] + select({ "@platforms//os:macos": ["//pkg/noun/platform/darwin"], "@platforms//os:linux": ["//pkg/noun/platform/linux"], diff --git a/pkg/noun/jets.c b/pkg/noun/jets.c index 00edefec40..00a6ae78eb 100644 --- a/pkg/noun/jets.c +++ b/pkg/noun/jets.c @@ -14,7 +14,7 @@ #include "retrieve.h" #include "serial.h" #include "trace.h" -#include "urcrypt/urcrypt.h" +#include "urcrypt.h" #include "vortex.h" #include "xtract.h" diff --git a/pkg/noun/manage.c b/pkg/noun/manage.c index 9b6d47fcab..0d425e3358 100644 --- a/pkg/noun/manage.c +++ b/pkg/noun/manage.c @@ -20,7 +20,7 @@ #include "platform/rsignal.h" #include "retrieve.h" #include "trace.h" -#include "urcrypt/urcrypt.h" +#include "urcrypt.h" #include "vortex.h" #include "xtract.h" diff --git a/pkg/urcrypt/.gitignore b/pkg/urcrypt/.gitignore deleted file mode 100644 index b464b3f2f1..0000000000 --- a/pkg/urcrypt/.gitignore +++ /dev/null @@ -1,56 +0,0 @@ -*.pc -config.h -config.status -libtool - -# the following was adapted from -# https://github.com/github/gitignore/blob/991e760c1c6d50fdda246e0178b9c58b06770b90/Autotools.gitignore - -# http://www.gnu.org/software/automake - -Makefile.in -build-aux/ar-lib -/mdate-sh -/py-compile -/test-driver -/ylwrap -.deps/ -.dirstamp - -# http://www.gnu.org/software/autoconf - -autom4te.cache -/autoscan.log -/autoscan-*.log -/aclocal.m4 -build-aux/compile -/config.cache -build-aux/config.guess -/config.h.in -build-aux/config.log -build-aux/config.status -build-aux/config.sub -/configure -/configure.scan -build-aux/depcomp -build-aux/install-sh -build-aux/missing -/stamp-h1 - -# https://www.gnu.org/software/libtool/ - -build-aux/ltmain.sh - -# http://www.gnu.org/software/m4/ - -build-aux/m4/libtool.m4 -build-aux/m4/ltoptions.m4 -build-aux/m4/ltsugar.m4 -build-aux/m4/ltversion.m4 -build-aux/m4/lt~obsolete.m4 - -# Generated Makefile -# (meta build system like autotools, -# can automatically generate from config.status script -# (which is called by configure script)) -Makefile diff --git a/pkg/urcrypt/BUILD.bazel b/pkg/urcrypt/BUILD.bazel deleted file mode 100644 index 22108ea72c..0000000000 --- a/pkg/urcrypt/BUILD.bazel +++ /dev/null @@ -1,36 +0,0 @@ -# -# LIBRARIES -# - -load("//bazel:common_settings.bzl", "vere_library") - -vere_library( - name = "urcrypt", - srcs = glob( - [ - "*.c", - "*.h", - ], - exclude = ["urcrypt.h"], - ), - hdrs = ["urcrypt.h"], - copts = [ - "-Wall", - "-g", - "-O3", - ], - include_prefix = "urcrypt", - includes = ["."], - linkstatic = True, - visibility = ["//pkg:__subpackages__"], - deps = [ - "//pkg/urcrypt/ge-additions", - "@aes_siv", - "@argon2", - "@ed25519", - "@keccak_tiny", - "@openssl", - "@scrypt", - "@secp256k1", - ], -) diff --git a/pkg/urcrypt/README.md b/pkg/urcrypt/README.md deleted file mode 100644 index 7170a906b3..0000000000 --- a/pkg/urcrypt/README.md +++ /dev/null @@ -1,33 +0,0 @@ -What is urcrypt? ----------------- -urcrypt is a library of cryptography routines used by urbit jets. - -Why is urcrypt? ---------------- -Urbit's C runtime (long the only urbit runtime) has accumulated a collection of -cryptography dependencies, some with custom additions or patches. These -libraries have different conventions and have been managed by u3 in an ad-hoc -manner. Reproducing that arrangement in other runtimes is tricky and -error-prone. The (sometimes inconsistent) logic must be reproduced and suitable -cryptography primitives must be found (or worse, written) for the new -environment. - -To ease these burdens, urcrypt isolates the quirks behind a consistent calling -convention. Everything is a little-endian byte array, and each jetted operation -has a corresponding function in the library. Jets simply unpack their nouns, -call urcrypt, and pack the results. - -What is a cryptography routine? -------------------------------- -This is more of a subjective question than it might appear. Any of the following -conditions are sufficient, but not necessary, for a function to be included in -urcrypt: - - * The routine is sensitive to side-channel attacks (encryption, etc) - * Some property of the routine is cryptographically useful (SHA, RIPE, etc) - * The routine typically lives in a crypto library, for whatever reason. - -A word on OpenSSL ------------------ -It is the library user's responsibility to initialize openssl, set custom memory -functions, etc. diff --git a/pkg/urcrypt/aes_cbc.c b/pkg/urcrypt/aes_cbc.c deleted file mode 100644 index f163bccd8c..0000000000 --- a/pkg/urcrypt/aes_cbc.c +++ /dev/null @@ -1,181 +0,0 @@ -#include "urcrypt.h" -#include "util.h" -#include -#include - -static int -urcrypt__cbc_pad(uint8_t **message_ptr, - size_t *length_ptr, - urcrypt_realloc_t realloc_ptr) -{ - size_t length = *length_ptr, - remain = length % 16; - - if ( 0 == remain ) { - // no padding needed - return 0; - } - else { - size_t padding = 16 - remain, - padded = length + padding; - - if ( padded < length ) { - // size_t overflow - return -1; - } - else { - uint8_t *out = (*realloc_ptr)(*message_ptr, padded); - if ( NULL == out ) { - return -2; - } - else { - memset(out + length, 0, padding); - *message_ptr = out; - *length_ptr = padded; - return 0; - } - } - } -} - -static int -urcrypt__cbc_help(uint8_t **message_ptr, - size_t *length_ptr, - const AES_KEY *key, - uint8_t ivec[16], - const int enc, - urcrypt_realloc_t realloc_ptr) -{ - if ( 0 != urcrypt__cbc_pad(message_ptr, length_ptr, realloc_ptr) ) { - return -1; - } - else { - uint8_t *out = *message_ptr; - size_t length = *length_ptr; - urcrypt__reverse(16, ivec); - urcrypt__reverse(length, out); - AES_cbc_encrypt(out, out, length, key, ivec, enc); - urcrypt__reverse(length, out); - return 0; - } -} - -int -urcrypt_aes_cbca_en(uint8_t **message_ptr, - size_t *length_ptr, - uint8_t key[16], - uint8_t ivec[16], - urcrypt_realloc_t realloc_ptr) -{ - AES_KEY aes_key; - - urcrypt__reverse(16, key); - - if ( 0 != AES_set_encrypt_key(key, 128, &aes_key) ) { - return -1; - } - else { - return urcrypt__cbc_help(message_ptr, length_ptr, - &aes_key, ivec, AES_ENCRYPT, realloc_ptr); - } -} - -int -urcrypt_aes_cbca_de(uint8_t **message_ptr, - size_t *length_ptr, - uint8_t key[16], - uint8_t ivec[16], - urcrypt_realloc_t realloc_ptr) -{ - AES_KEY aes_key; - - urcrypt__reverse(16, key); - - if ( 0 != AES_set_decrypt_key(key, 128, &aes_key) ) { - return -1; - } - else { - return urcrypt__cbc_help(message_ptr, length_ptr, - &aes_key, ivec, AES_DECRYPT, realloc_ptr); - } -} - -int -urcrypt_aes_cbcb_en(uint8_t **message_ptr, - size_t *length_ptr, - uint8_t key[24], - uint8_t ivec[16], - urcrypt_realloc_t realloc_ptr) -{ - AES_KEY aes_key; - - urcrypt__reverse(24, key); - - if ( 0 != AES_set_encrypt_key(key, 192, &aes_key) ) { - return -1; - } - else { - return urcrypt__cbc_help(message_ptr, length_ptr, - &aes_key, ivec, AES_ENCRYPT, realloc_ptr); - } -} - -int -urcrypt_aes_cbcb_de(uint8_t **message_ptr, - size_t *length_ptr, - uint8_t key[24], - uint8_t ivec[16], - urcrypt_realloc_t realloc_ptr) -{ - AES_KEY aes_key; - - urcrypt__reverse(24, key); - - if ( 0 != AES_set_decrypt_key(key, 192, &aes_key) ) { - return -1; - } - else { - return urcrypt__cbc_help(message_ptr, length_ptr, - &aes_key, ivec, AES_DECRYPT, realloc_ptr); - } -} - -int -urcrypt_aes_cbcc_en(uint8_t **message_ptr, - size_t *length_ptr, - uint8_t key[32], - uint8_t ivec[16], - urcrypt_realloc_t realloc_ptr) -{ - AES_KEY aes_key; - - urcrypt__reverse(32, key); - - if ( 0 != AES_set_encrypt_key(key, 256, &aes_key) ) { - return -1; - } - else { - return urcrypt__cbc_help(message_ptr, length_ptr, - &aes_key, ivec, AES_ENCRYPT, realloc_ptr); - } -} - -int -urcrypt_aes_cbcc_de(uint8_t **message_ptr, - size_t *length_ptr, - uint8_t key[32], - uint8_t ivec[16], - urcrypt_realloc_t realloc_ptr) -{ - AES_KEY aes_key; - - urcrypt__reverse(32, key); - - if ( 0 != AES_set_decrypt_key(key, 256, &aes_key) ) { - return -1; - } - else { - return urcrypt__cbc_help(message_ptr, length_ptr, - &aes_key, ivec, AES_DECRYPT, realloc_ptr); - } -} diff --git a/pkg/urcrypt/aes_ecb.c b/pkg/urcrypt/aes_ecb.c deleted file mode 100644 index 9b39100126..0000000000 --- a/pkg/urcrypt/aes_ecb.c +++ /dev/null @@ -1,111 +0,0 @@ -#include "urcrypt.h" -#include "util.h" -#include - -int -urcrypt_aes_ecba_en(uint8_t key[16], uint8_t block[16], uint8_t out[16]) -{ - AES_KEY aes_key; - - urcrypt__reverse(16, key); - urcrypt__reverse(16, block); - - if ( 0 != AES_set_encrypt_key(key, 128, &aes_key) ) { - return -1; - } - else { - AES_ecb_encrypt(block, out, &aes_key, AES_ENCRYPT); - urcrypt__reverse(16, out); - return 0; - } -} - -int -urcrypt_aes_ecba_de(uint8_t key[16], uint8_t block[16], uint8_t out[16]) -{ - AES_KEY aes_key; - - urcrypt__reverse(16, key); - urcrypt__reverse(16, block); - - if ( 0 != AES_set_decrypt_key(key, 128, &aes_key) ) { - return -1; - } - else { - AES_ecb_encrypt(block, out, &aes_key, AES_DECRYPT); - urcrypt__reverse(16, out); - return 0; - } -} - -int -urcrypt_aes_ecbb_en(uint8_t key[24], uint8_t block[16], uint8_t out[16]) -{ - AES_KEY aes_key; - - urcrypt__reverse(24, key); - urcrypt__reverse(16, block); - - if ( 0 != AES_set_encrypt_key(key, 192, &aes_key) ) { - return -1; - } - else { - AES_ecb_encrypt(block, out, &aes_key, AES_ENCRYPT); - urcrypt__reverse(16, out); - return 0; - } -} - -int -urcrypt_aes_ecbb_de(uint8_t key[24], uint8_t block[16], uint8_t out[16]) -{ - AES_KEY aes_key; - - urcrypt__reverse(24, key); - urcrypt__reverse(16, block); - - if ( 0 != AES_set_decrypt_key(key, 192, &aes_key) ) { - return -1; - } - else { - AES_ecb_encrypt(block, out, &aes_key, AES_DECRYPT); - urcrypt__reverse(16, out); - return 0; - } -} - -int -urcrypt_aes_ecbc_en(uint8_t key[32], uint8_t block[16], uint8_t out[16]) -{ - AES_KEY aes_key; - - urcrypt__reverse(32, key); - urcrypt__reverse(16, block); - - if ( 0 != AES_set_encrypt_key(key, 256, &aes_key) ) { - return -1; - } - else { - AES_ecb_encrypt(block, out, &aes_key, AES_ENCRYPT); - urcrypt__reverse(16, out); - return 0; - } -} - -int -urcrypt_aes_ecbc_de(uint8_t key[32], uint8_t block[16], uint8_t out[16]) -{ - AES_KEY aes_key; - - urcrypt__reverse(32, key); - urcrypt__reverse(16, block); - - if ( 0 != AES_set_decrypt_key(key, 256, &aes_key) ) { - return -1; - } - else { - AES_ecb_encrypt(block, out, &aes_key, AES_DECRYPT); - urcrypt__reverse(16, out); - return 0; - } -} diff --git a/pkg/urcrypt/aes_siv.c b/pkg/urcrypt/aes_siv.c deleted file mode 100644 index 7e06e769f2..0000000000 --- a/pkg/urcrypt/aes_siv.c +++ /dev/null @@ -1,181 +0,0 @@ -#include "urcrypt.h" -#include "util.h" -#include "aes_siv.h" - -static AES_SIV_CTX* -urcrypt__aes_siv_init(uint8_t *key, - size_t key_length, - urcrypt_aes_siv_data *data, - size_t data_length) -{ - AES_SIV_CTX *ctx = AES_SIV_CTX_new(); - if ( NULL == ctx ) { - return NULL; - } - else { - urcrypt__reverse(key_length, key); - if ( 0 == AES_SIV_Init(ctx, key, key_length) ) { - AES_SIV_CTX_free(ctx); - return NULL; - } - else { - size_t i, len; - uint8_t *dat; - - for ( i = 0; i < data_length; ++i ) { - len = data[i].length; - dat = data[i].bytes; - urcrypt__reverse(len, dat); - if ( 0 == AES_SIV_AssociateData(ctx, dat, len) ) { - AES_SIV_CTX_free(ctx); - return NULL; - } - } - - return ctx; - } - } -} - -static int -urcrypt__aes_siv_en(uint8_t *key, - size_t key_length, - uint8_t *message, - size_t message_length, - urcrypt_aes_siv_data *data, - size_t data_length, - uint8_t iv[16], - uint8_t *out) -{ - AES_SIV_CTX *ctx = urcrypt__aes_siv_init(key, key_length, data, data_length); - - if ( NULL == ctx ) { - return -1; - } - else { - int ret; - urcrypt__reverse(message_length, message); - ret = AES_SIV_EncryptFinal(ctx, iv, out, message, message_length); - AES_SIV_CTX_free(ctx); - - if ( 0 == ret ) { - return -2; - } - else { - urcrypt__reverse(16, iv); - urcrypt__reverse(message_length, out); - return 0; - } - } -} - -static int -urcrypt__aes_siv_de(uint8_t *key, - size_t key_length, - uint8_t *message, - size_t message_length, - urcrypt_aes_siv_data *data, - size_t data_length, - uint8_t iv[16], - uint8_t *out) -{ - AES_SIV_CTX *ctx = urcrypt__aes_siv_init(key, key_length, data, data_length); - - if ( NULL == ctx ) { - return -1; - } - else { - int ret; - - urcrypt__reverse(message_length, message); - urcrypt__reverse(16, iv); - ret = AES_SIV_DecryptFinal(ctx, out, iv, message, message_length); - AES_SIV_CTX_free(ctx); - - if ( 0 == ret ) { - return -2; - } - else { - urcrypt__reverse(message_length, out); - return 0; - } - } -} - -int -urcrypt_aes_siva_en(uint8_t *message, - size_t message_length, - urcrypt_aes_siv_data *data, - size_t data_length, - uint8_t key[32], - uint8_t iv[16], - uint8_t *out) -{ - return urcrypt__aes_siv_en(key, 32, - message, message_length, data, data_length, iv, out); -} - -int -urcrypt_aes_siva_de(uint8_t *message, - size_t message_length, - urcrypt_aes_siv_data *data, - size_t data_length, - uint8_t key[32], - uint8_t iv[16], - uint8_t *out) -{ - return urcrypt__aes_siv_de(key, 32, - message, message_length, data, data_length, iv, out); -} - -int -urcrypt_aes_sivb_en(uint8_t *message, - size_t message_length, - urcrypt_aes_siv_data *data, - size_t data_length, - uint8_t key[48], - uint8_t iv[16], - uint8_t *out) -{ - return urcrypt__aes_siv_en(key, 48, - message, message_length, data, data_length, iv, out); -} - -int -urcrypt_aes_sivb_de(uint8_t *message, - size_t message_length, - urcrypt_aes_siv_data *data, - size_t data_length, - uint8_t key[48], - uint8_t iv[16], - uint8_t *out) -{ - return urcrypt__aes_siv_de(key, 48, - message, message_length, data, data_length, iv, out); -} - -int -urcrypt_aes_sivc_en(uint8_t *message, - size_t message_length, - urcrypt_aes_siv_data *data, - size_t data_length, - uint8_t key[64], - uint8_t iv[16], - uint8_t *out) -{ - return urcrypt__aes_siv_en(key, 64, - message, message_length, data, data_length, iv, out); -} - -int -urcrypt_aes_sivc_de(uint8_t *message, - size_t message_length, - urcrypt_aes_siv_data *data, - size_t data_length, - uint8_t key[64], - uint8_t iv[16], - uint8_t *out) -{ - return urcrypt__aes_siv_de(key, 64, - message, message_length, data, data_length, iv, out); -} diff --git a/pkg/urcrypt/argon.c b/pkg/urcrypt/argon.c deleted file mode 100644 index e1090dddec..0000000000 --- a/pkg/urcrypt/argon.c +++ /dev/null @@ -1,120 +0,0 @@ -#include "urcrypt.h" -#include "util.h" -#include -#include - -// library convention is to have sizes in size_t, but argon2 wants them -// in uint32_t, so here's a helper macro for ensuring equivalence. -#define SZ_32(s) ( sizeof(size_t) <= sizeof(uint32_t) || s <= 0xFFFFFFFF ) - -const char* -urcrypt_argon2(uint8_t type, - uint32_t version, - uint32_t threads, - uint32_t memory_cost, - uint32_t time_cost, - size_t secret_length, - uint8_t *secret, - size_t associated_length, - uint8_t *associated, - size_t password_length, - uint8_t *password, - size_t salt_length, - uint8_t *salt, - size_t out_length, - uint8_t *out, - urcrypt_argon2_alloc_t alloc_ptr, - urcrypt_argon2_free_t free_ptr) -{ - if ( !( SZ_32(secret_length) && - SZ_32(associated_length) && - SZ_32(password_length) && - SZ_32(salt_length) && - SZ_32(out_length) ) ) { - return "length > 32 bits"; - } - else { - int (*f)(argon2_context*); - int result; - - switch ( type ) { - default: - return "unknown type"; - case urcrypt_argon2_d: - f = &argon2d_ctx; - break; - case urcrypt_argon2_i: - f = &argon2i_ctx; - break; - case urcrypt_argon2_id: - f = &argon2id_ctx; - break; - case urcrypt_argon2_u: - f = &argon2u_ctx; - break; - } - - urcrypt__reverse(secret_length, secret); - urcrypt__reverse(associated_length, associated); - urcrypt__reverse(password_length, password); - urcrypt__reverse(salt_length, salt); - - argon2_context context = { - out, // output array, at least [digest length] in size - out_length, // digest length - password, // password array - password_length, // password length - salt, // salt array - salt_length, // salt length - secret, // optional secret data - secret_length, - associated, // optional associated data - associated_length, - time_cost, // performance cost configuration - memory_cost, - threads, - threads, - version, // algorithm version - alloc_ptr, // custom memory allocation function - free_ptr, // custom memory deallocation function - ARGON2_DEFAULT_FLAGS // by default only internal memory is cleared - }; - - result = (*f)(&context); - - if ( ARGON2_OK != result ) { - return argon2_error_message(result); - } - else { - urcrypt__reverse(out_length, out); - return NULL; - } - } -} - -int -urcrypt_blake2(size_t message_length, - uint8_t *message, - size_t key_length, - uint8_t key[64], - size_t out_length, - uint8_t *out) -{ - if ( key_length > 64 ) { - return -1; - } - else { - urcrypt__reverse(message_length, message); - urcrypt__reverse(key_length, key); - - if ( 0 != blake2b(out, out_length, - message, message_length, - key, key_length)) { - return -1; - } - else { - urcrypt__reverse(out_length, out); - return 0; - } - } -} diff --git a/pkg/urcrypt/argon2/argon2-specs.pdf b/pkg/urcrypt/argon2/argon2-specs.pdf deleted file mode 100644 index d916af6415c93ade373b61c9179f9928b2b1daa9..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 459608 zcmb5VLy#~)w`E(lZQHhO+phY`wr$(CZQHhOoBel3M|8jFJGd`LCvudt6>FcJqzWQp zw2XADP^9zABkNF%oU8;41a?N21Ux)Y^fIQl=FS!bjI2zY1pn(m(Tn}Fb~be+pck_? zbT$<+HMTP`h2rCba&mSwHMD_p-@MV1cG_Y?=si&Ez7Lk8J5Ac_Npe+Gzg}5JXL;?^k@b=5P_JyvWx8mg=9j;j!NIQmq`Gp}%^*L} zH6*vTTs(PP-)*vRR@D3*VC@v@>AOkip)m`?98tmeC=) zJ0tDwld7hLddl}WTe`fI^6y6f5Iej2LhZ6v_uxmKQ_Err&d!<5v>|P)Wjq`NlD&yI z!e07A;{tmLv9yGH$t7gA>+aYt5(2{lfs-XkbW}hQa{gjW5U$7lo20 zM`u%YjlJ$mDru008sz+}M^o-@?&fepNBze(?5ewUDyZb74&Ag8I&q~$*I>}UYB=UN zbh4{fvo~|b*3M3HwKlVs?vq=oEh?k-lmWWGWg-jnGo$4YoJ6uXh;p-~ab9;*e z0Gbr%0t;dX1f_ld1HNp%Mm0n#ZE}{$gaks!N(~~yb+63EtbKL?0(llvLx6aUOGXxf zj-(L|Rs^iS(|`nW+fv?8E!#6^Fe(ZcPaY{zf@~&iz_xgyBCmel3ml)J&dQ3JsvTmq zJ)=6Dq#$w7t+mhC-o48>s-TAXMO;9%WNo3#F6gMa)%*lnQm~ZhXa>yJJihRD*d*V1A+Op_d zXWiP6IRw?GcLGV;9cF_$_^6N3Oc0e=5Q3IK(TLrfBXq7VCPpyjAF^16lRKW1O1g%- zwo)g$=HdMK;nldOIx8hNLVR#gPc0K z2IwVG4^1*PTPUJ8-hELgZ=p)Wn2^HRq~?76H?>QYTg6kBNlS z<|`$yX$RgH!NE{$byg zk*x?oc)@6xaIj@Sot=%x&rxRwrPP~M^eyt7!-+YpUYc7-f61T*UCDO3J}^W_@bpoq zIt21IY|(@A2Acw^l}qcrB-zN=AUg{7ID)R%rF(tCEvy`{0f7oMFQp!MHe>yixJ3t& zlK_cJKl?>+`)GtQ+6w@nQMY++@f(S5+&qf)>vv#Lq`OjJPTp_-% z#(dkuraBB9G$yR@e*|0{N`d;m}5Q0I6M(OHpu3qd|idRQ8!**r~A+%b#ko5L>{c_T|@XB#MbIRZuohW|+tM<-_j4hGi$!x?tY|2Ah@G`4KFR*`(x z)TfQ1mXCY&;i7?Jqni0RL}QA5q<#dHn?*#3KqC`xe0|mwwB?qJlWeO1kk~|Lsx_T9 zn&=ZjG#3MLFr^c^Nhv8r5;KdLNFEk)N&HYI1f)2W5|dq-OC&hGX)DejATYksC9x;M zH!T<6gZ6(412b{(Y6#C5-5-G# z1G-a>Nj%QX_793eFhE^L74H=yQ9y@5M28kpg2>?Gf`8#;{DV*cSoEfg0SXP>>r>SpfEjAQ z^844Ah(UC@@}*+CmcISlW&x6o@4(aqUZW=w?tQ>~-5F#M#&@59dky0O9sRQsD?mil znJf%2IjDtI!iL30d;vI#29Y=u7y&Rf1ra#@#%m22(iH<) z>1>cIMeYcuCL0Awv@6Zd?UPD{d=NNM)ywM$BW{H56@U1>%W5^a469;~FqP9?aUkKD z4nG88;;vIPRu{!F4}-C?fcGAe zNz)*AknLQ5XFtLx|IPm{D*f{IY^a&)x5<+A72Ne^^>kQ%*5=%Wy6tsSRKpGKsk^x- zKV{u8H+F52KM;S)+vwj#q1XS>)WtqNS&F*Zh+kRmxW<`J6=p5%)Kcm*Wg@@3tj$c(~sKdK7rN^^@Urf37TvXnq z`fz;8@poT)D|xA-{=j(_iA^tTMfoV!MQ*8zy45X7=jyi+u$A9vL{*0MTDBf-G*EJ@ zc3fTlCPbEM@Y-HEM7@qb`z&}1U6q&`=b{RqVRBa{2ixpwlbK zV{&60M{2(GmH6l_e9ZWk$`aE!{5)XMFeZ_{H?_1&nUT7VsI+cobO^1=(7` z*N#SKO_c5)( z_his9;JQnZ2WtJ3lbZ2@<%sXv#>?>Zw=z@qefaHNAFmI?n&da`=i@iB$3E4&XZ@FS z_V)Isxxs7u>DuBozG<7Vwtd<(=e=^eVTPPl(QK*$2^ z$zc(Yf@fMFH#QhG{g4ybpeb{8S)WXIT#Kg ztE^Zy7lV>f@6G8i<*>~qf);{16T{~r;0&{2PhaS4Mymnw41!w*RM;nE>)gMZS|{f$ za`&7cS_tczIe9^?=9NwH@hd#E(En!4hFljhsV)&O{P3^$VxU$8L}4id!YnhA(__R^ z1}!9&W+5_U*p6Dzv^y((30x#f<|&kDY5m6nB`VR83g^^nxe6u&kM0O)&`{F&MtxOQ znbU_}als^%o=(^7qf~CcxNgY8=z8L)AaL@1P(w@(($iWbvh=L(2dY95qvq^OHsBWR zU5$Cj*8D#&(~fA=ctQ3o6w7wlt5ZN9in!Cl)5HV?bJ9J1F^>=kf3(Y-~e@6>* zmY37(gFhU?cD6@kN!HB@--)WOxa)6>AJ~n(57|^Zg*4xcv-|@V{>Rtg?~DWZ7z4ai^j z^pbRCW%<&4ffs~x>+)^|- zQ*@*5JCVqy9fo-k9PFvSx+|LsPXFx0u0gmRu(EYsea^cKYWPiUvc*koZl;$=P8hHd z=h_OAxt@Q?ewxQ1H<0__TpS|30woRtWIpvwj}fnD-8KD_+0#l9_E6RWmXA@o`&rOv z((~x<$rEP-4Bfdgcm@r79aM6PcVs1vT-oaD()lAM#N~G2;nNj1q6nSk3#xx;rTy(w`U{ZW~+tHL0 zs6POd zLL0a}mpK10e*+8>Emqvbcy4;O>m4Y$;wV&IE{6bNq*sn{+zF&Uy4JNYwk8LB`#A5W zHm3Ts#1}5pmnc_ygG>Jn%>oYCo&*HkMG#G)E4Rw!1@9#ioUTAE_K>^j4C43PJ06&Y%uL*GyrYHibW`7N6>Bhv(Nww46K-`L?37k zkL)JD&HZn1E$=uU3_Fxk%cu@7tALN)j9L=w5;+J;RNt2u)Uva+hZ566z_bS8V}3(W zq8`(C&gvY6t&>^@bAgL+j~J6#3}QQ+Q|DO=>j?InnD$r^My}LEIH?o6*qSiKXz9?1 zJVgQS=TcNHWZ8{Zm+i!mzhrN9TlnCZsYjzRGz!UI%O?&uCuBgcvVKi{uh$+&n)MD& zx9;l`jn7SV@?)X$$OcC>n@G^024Oo|PQTmh394wF8?sZCSkjWCwf>28KJ-8#o`V#) zNFMiKrjE$0QM=F;I7mwOHY0X>u5J}OUStYeplOp1KW7_HA%+_edYgMnX%orIE|k`} zGp#*J*c8Ihh!O-1VN;GqWavQS(CDlJSY6ND=>$%lZ^dUV7X?&>Y)l;Hc036jbJqz$ zSyL#Jt~pGoBVt$UJf72cw<-eawqz!7q2ze!k5#Eq9P`EfB_~Z%FeqXX#qGC1%37s}ux&glI0ZL$^#7h{D zP%bB`>hyMBMiU3e$>DMvH+wAFWCCTzlxUPKrB~GscjPjOqPK~9RLP6S+vWv;`dGyY zF-`+nK%JUtP#NT*)JD|PD_l&lQYaFA49jU%LkkkrEglyT*{gpuSt~bM%e3NzVd{rv zs;uAV9Df$OfP4Cf!d?bH@Yu7bqjANDn=Iv?P>^^PAv@FP>5;)NFp#FwxfVFE9G_)!ORDz zvkAp3j;p8K4w;|+`BiDgekmQfpFtXwmpPK6Z&_`D-Pl;P0PE@>ST>3;lYTKe&~oyR z_u)4<@)m1RYGptte*6nhN@UeEU#=YwS&Ob<-;?IIZhh4`r9_ZRZ_#)66pwQs^^?!r zE2xHkLRaX*t{1R8r0yarQINix+Vqhn<(PnCNK;jP`WrmoG7hfz1Og?8#|{CsCz4O- zh&YAY_RQU-l;k-j&u%Q%OpoCCQV7_`Y%`8<*Xu{>SYasrUHb)k7X?C+xh`1bxr*hK zWuc25$`Ir;dg#zTnj{1m$X}_Q^frq1ps%Q<{6S;jO8g9WA;#D@U7B)$I{n}K%$0&0 zbNgDiHAz~bGhyo(F4dnU@WX#F-=D*04{_@Q_o~cKS&Grk@vGdRVvFvJ1%BoJw&5;B zp~2CCv-+P5*~Pm8kXjt7T89cX*^&hw`>v=Sv24RGT>HRzxsgvZfnEaWuIesU$dNPt z8TDU86YEBF0-(OLf%#Km+GI{9Q4$5RkQMF7Mkjg3kqZK+b5jC5c}}6Tc$d}(4RNy3#`7zzaGe02T%m_ z2O@{6;9dZv%mo+iQR7jw`Hr6i+QVxSBrD?S{J5jmLdhhuUG5vOg+NL#TcQPqJ^m)9 zO7IQjz4{R6qO6bgNT{v3F}(`3cLUO~EAR}-?c&M1XqhC*Gth5xN6iz>KzU;diXGO^ z`{Y=o8PffY>~%w#)@3@vbTX@{!hODTrZ9d^{msa7=s74#zdr2&Xv z4zKtY;SDO}2qcujX&a0q?TEkG4#7lM$b?Ec2yx0#$Gf?m7zM5rFK*C!L5_WaM9KCa zO17m;q70eStd5PilfpiV)QVgL4k?tY)OlMt_GTl}Q$n?)#XH4}2SBKQxaSJclY{^2 z-50qj*u20)l&SStmvJ&IICcPJ)X+D|X?})(%CD3m5(JIk|Y{-ALNP!NpJgqfeEt7gu zi)c!=X^e=H$qgT{0|lAHfmynp;Y+0QK4o}Ph!86lUt=MKt%#M~dXnQe6Fr zY;h$1rI%!WkGHQuDW;Jy=L1rzfo(#vO!{$8x`ANPAz8KZCirfWdmZiZ?p0nr4%5H( zk@A@>Ag9EiyV+gomljQzu+`9{`vQE(YiAddJQlFTu1Yz^A+rz~4*cur{*W+&AetYC zD_r4@a{id-uJo9+|1+hdHsg3N4h4vb3fC5&9@$kb&Q*(NIeg4jlUn>8suK$j*8d(< zz+%(2LmQ}CBPAzR#(Yk9{hn5FK|69obXV;!Log40*T8@Ihs@Ad~H zp(u#5AMynXRr9SG@S8cWDeNF{IL2Bdkc9+~amebpCPnc%*u;0gE_Y;QRgmng@^DEf zdC>JAR#n*spxibbXe)62C+Y8x!ar6vvJKs6MHq1h4!RaygR4W%xJ|t;CV1dj*F#>z z8edL>;Z-uZqQt566^qD2*piFOm zIn!T_YJF^4tPU|qEl|F>pu7fY~WKKP^(!8c3Px|()aE(?I(&dfURaUxy%GUcS z0CMRzHM&qGF6Ld^LCTb?l_gc1c>B_yz(M(|Xf!u^>WWgb2CHO^zM~fNvo%Y~yOgCH z`LZ&~Z#ICy0{tT3EZ$h=AG+|2d5JsH8`@Xi*)Brp+eyGudM_@1S%Pe2iardjQ?nIm zo3G$MG+c%&(I2C5mW#-%(st&se!#ePTvb{pV{n>ieCRa^ngSZoi0Ifbw)^bz@{)?( z$SJ&t!BBSA${fhc@0^PSV>4*@FXt*)lE*Z3mIPzk$13BkwAy6Z0>l{-jc1#euWUo+ zY!#WOIgxsRlSkOAJP%li&QMMZKVJD^Y}%h6f+sv)-Z&`nyL7{rupqO4Gyz?^GAiES zDXf?iF_!ZPGxbj*9%TvK?3QstIQyOoLP9KhC5UE-Jf=0^1y|hXSEXvs!0`yB7!!n} zY_(mPk$*NF7D=6cQFKROaTVjDK^gU)u=g~Q;>ttumS28t*Tiksxf%C1)|<=h9zb%A|Hz$a3i~%Dz3J0|x}w3pk3T^kgmQ0Y z_H)_z?(x%ZH2?Q`Iny=8UJ2t=yV{{G6PYx+`9)IHljtl|C`{_~&vjg8QuQ9x4#rOV zlmnC9=7U2|$J};1g)~}vn5SAJ*^Sl$)xeuYSltAuQt=HAO~m(;MRg{-^eMeZBgtQM zB5j}Z#CA_26yuLTxp$Gru)T2$ZPLF@Ij<~ii+!C|y=TebzERuPuha|k%&Ab;=l6tIN6Va${J4*cLxyJf=E-uIy|`d_GT z;Dh`BLfct5{$FipVPa?g?`nIG)_tH*R{>_EZu>M`uTu6fk|n zViikF4+E1r`^w7K4*xen&#(Pc8h!698gB)P?#JSZsvE9ZSKQhc#f#g}3*S0!ZIG^; zEmb9y>(PLe_D=5XQLinvSLbfn#cbiSUJ-+om!fJ9E*SUt6)l%3_fP9}?EAW!U+?ai zL${Z?Up~BzJ_}b}%UEZR9p1K;$8F8@Ue0KzmCL-aVARjSW(>|f3kD?h09!+u4}!Y~ z&eU$C1C7+>H~nv*D@w+W*V6!ULG^JPdc$?lw)E0@UxI$URphT8x*83MRvW^FPZ6!S z*Yzj3uJ2MqwsVROJC51K)vJc>#jUos3PO@AH=3-mm&Zj^YoRgsqVt!sa&oseSJ(6a zp6E*gy{K1 zBQd8~e8Z9pvWa8;Il*EYVCR4y46LjWFWdZMg0i0hARuV97S-*njU*c!sr;kKo7_rX zAD&Bq5g3!h27mVGZ7;nX!a_mlApkp@+&`fNgv+HV>xM=x| z`BJs4B15fEie%fU0y>xJ&!z1HIkB`OanPbu{~Ao>+g%hISlq(knN?LrCZj{2@;GvK zo>M-~2sQ0j_2xo``I=sgD@ddQPSSnH>p2?b3z2<-kl){HgZg%QOXuWJ%~H91ofrod z!h((d^n==~I8flnKOmu@>3Xe&C0{O9+#LR%`jLJLQ26rFVL>=0k_;#Eg#oBeGF1O? zKy%Dfv$qF>79{c~0#Y_lx^U%)0ALs8Z8nfLFjc#X-g8K^tWR-1<6JD%AXaf{=&&L4)RVV`>CP|KuG z^$jVp_ZIyE1rT4t3+hS-$>s#)=mQf8<`^Y0F2Hj_c=ejvI9Z)7pEVkOxzkjEjn%_3 zrNpQ&3o#|EjgLI0N)X=7Lq>-Y2~kvY*L+0! z-Mmz*uJWvdBp!Ibg*D=h4#-2ra4lD?(Jo684?rUom5pw#xXo7}LC=jWIJViaVXr+U z>SS=;EU~_lO|P|kQaFP{NMs%26j)Ad?jo-070CQhd5t2Ok$N=X?fS7&i93iz`Q@Av zXBSTq#1{m$UoxT-qPyreKgI?5NtMjna7$Z@P+XEjRw0#UdLyG5{9aE;H=5PWkTCup zGYrniRhp74&^&u_{6i{F%Pv5z<0wt+cf~;Ua3Ggf58X%iC#%7HM5WNyK$pO=vH;?p-?>>5`J*CO!P5@`W?kuA@=GB#>tI1tUODaM9eAyjwoIchfI-& z*?mOn2P6MF$*|-^2VqVXkd92#G7~x?loqP`xY3YIU>8++*l8Yg;*OP0Vo2FT3bHMszrGu;LM6|H?zNOOrR{a*qHF*e#!QMQm&}G2G_;Xl9QNfioTCr9XaW`$)^RNO{ZA5+|5vlm-Yy)S ze(J$)$^UTt!w?4#{^-uUFQ}LMH@V}FXBpowZg9BGT%k#f%6za+fm-0$3|)Mgb@pfy z7kX)zw*a?04&Bte&fqr~)Cze!!&6pD+csqf8DStQn_!gvY3p0y&lS?GaLN~nz+ zD0!(^kkIl{1}hPi9DNkAJc9hdNc#(0cLd9=eZVwyVBB z@dbEhd-ZGy+ymcWBM|=%=mvP>H$?$3ldW$2u8MlU7R89_kw?UCZ+-oWi{p>bb}DWoPr#dy)S z*11#gKvoB#GSw|RlS8=hInuZrC&y7zta+Du+z2w*dFGt1v7hib!?!b()ovr)58hO0 z+#Z5^Je@B44&=8U0^LX@#ayw(*bM#W32IDy+t?Q%fAshFgYRvtIeg%W@2b&b9a4P- z#Zk|fzv%cfb3)WOJ3Bt1hPLxzO_}Xg6=K1>gNXLCjIElmS)7>8^FdTnjE(Sh62TpN zDwv>I!PWhV*e0QjF+C$7lrwMo1*kyRBay#QA~(00XVO%-=o9F=m~!q@MYpU3nxN?% zd4yf3pcM*xDzGD1Y2p+-@v+>kKMAHBbz|&U9O#!c49ikwl*_7kPNdwH=UZ+q_4~gu zDjK=lDv5;CuWE)F&Q>ap6Xp@YMcI6RsKtinp6a;E)(Lb^#t$Lz-loF9DZL__29rpg znL^;2*bNOSTWNAbj)JnFd=~X+$tKl(GTAf1bzQ3R9zo2Flgmj!t_7uOY>O!RB1p(x z>vVhrsthIUZl~yq15_Mn?hF8Ir24p15MepVYkW*v;SL)M5&Uw|eHJYUoSgB7tIUZJ zUOk(16Fs(p(WV#_w!v`w*`f2rY8a_aH(PB)g59$VjUzmHakoO3lFE@wHK%+TyXAF5br++JeT08{Q=$eOe%wO^bKF1t{I{UC**+ztzdzyOnFBO}3x9-HLv5}mWz&Fu_X9u}Ti2O4)EO$+u>WF=AK+zt8lxbVWPLpaL)lLdY5W=^IWFJz7#o6e=5gctLud~D^ z0~gnn23d5-UC~u^;zicy`bowz?vjWKGM!d4$(=kL?B2;?OrGQ66Fr-ge#!*b1P(<1 zO$Ocfnhz>5ANsY+Cc^fR-Tn`W1)ua&3_OGE(xd6FYUEq=E zI~21W6d^!#K$5odBudI*hLd4r_VX;s?&UvDO*1_n2b|K10{75u2`2B+X^&mksG=bJ z;|&v0`c?v_EartgVR(EXOK3PKu&m^#74W)&HoE7)uZWLhRttVDsKyzRVVABUhi0h3 zbOy>n3LQJU^?>8ba=4BzJReoClek`n*)P28nWh}%qy1N zwDks{&0p{Tn$BVr2^Kl?f0tabNOM!MzLoOd$8JId6dT={j*QlcbPK@I?%nyRr$8}e zlz(iEGZDyGUG5=!nTEvxwt5YrK2&^pDqhZ6qejE{HPp|bpsg9vA?bc*2HgitpVcMl ztNiX-a)1;?B?)3eQs{aC3tgY|%YK%kK(1xbtlqBl7|&t#FB}_`W5fG6-eNs6C!jm} zI(oNFfU)`lJ%bO-|I}WLXEbM6GQVUQ zfw*=8MSZHP9_mwOPxUodOeFT7=&i;~orQ^NDm|f?AK+%NzdD`O<50X?&oVP)`CmS$X64#4wjbP`lNC4j5z_3mgO2?6@=o&5 z_xZe@B86@zKC@B(r9EAxOIJGBZlCc}LEY3_m;cng64_$Q^m_pWrIzV#)}18JazzwT zAOZ>Qf&1D26K;ISovThrQQurXdZs{|srF7f4HYj|P2F7jydtB2jD$_~`9A3)#tf%w zIbb<@XzN$2Ry4BPygY?WH)-i9iZ^jLztryPj}u{6HC()|*EO9Ane=jo=GAE4xh=b$ zcSW(zZ!8Z9@DACyS+}v+4@*W?m(s7KU~l4zd!EV6*YCux&#I@UbWp|Hl&~__GmTW& z4|ntE>kOzURRKfS3B7RN2_&;Y26nkR3|&OparIkB19-tQoN7xy1eS$ z!!rcA%U090n#d;VLAy&mP16Q&x~`O&g>~b(^%m{A{P4EaRsOuzZp5}dYuVWE+bH`v zKkj@AzB|fi-`eX(T5G@gLhk*&*A38{fs`Wx&FGP7^EtSJYjHulzh*8Dkc+bMJpbvSc`BfPal z9~Xt_EcVA{gnHrHRUv8s|dP3Z|Z)X z4N%(eaL*W^TWQ}b|CZ~%_=si+dJ+}B{7YiiMKk={D{jur;_>$T3y@#~uBdJe#Z*`c znGDRmXodB(UyIKTCWh9{vUNK@h?c{`4NfyqNKcfk0tVj8V}{78BLnqKA!txb+yH_L zLXnveN4G2sanz{T&o=jyP*BNEDSW`(8lz{pw8qwHPHoh|{Cg+0&9h zDEBDTBbAU&-TFp!Wo`Okc>(8v5p`?|4#^mf0E%D+1@e6)(9PYXw+CFUU43fT?q<0&-$tY8wU`HUwp%2;87aiq#DTZQ|oF-P^Wco=a) zr%#G%J3&^r>m9xv>Hc(5hu$8yP}PYK7MfC`E=b$&Y5t9LMEzDA_{~O=m^fKesba3t z%I11VR(|6mBoyHEAjD6gNv;+@t<5p`V|V;kMG**tW!Ru(HNL2n_42?JqnHK%^c2D7;|K2EnrbT#L z5{kNK*hjF1gn8ILB0uL)8L|%p<^rGIM>bIXWKp?!N6G|%t&9EjE8GM`>9D-3pyow| zFuW@Z?(n?i!l~1Ok!1UAxvbLVUHuUtc4z|pA_#xw&6+$Ye9AeU^4U-`Zn532j3JO4 z9pwP6+8=bhu(e5;oQpXTOZ z_;A>~w)|QcQzg0ztF*fucmljE8G;#l5b$roz61cTau{T2j}L*c5N`?nS!csOg!xBw z=!$&}2M@QtKip2`-4SwQ`vblWAMYm3FBgu{tAY)KsEFwZ8WzXJYZ6wlw4H@#oJCYr zUh|e?NY`J#_98JLPoRhlx-HPwTHoLbSQ)&gK^obQE&)|iDiyB%*(qE|vWmB^l4w4| z8iRj6=8Z`L|J?96oTb}?2e&dh$N0L^db?jtDGy_|m-n~;!>1IoTa_9Jr`1C)R~e*# z;Xg0E0=HvS;R`It75)@gti0GM;6G^N{*RjG5W?^uWr z2?Kz=J6-3k^om!quX*MHL1@`sXuc{^>xF*o^eBX}es}_@y7jInS;J|f9{)wQNXh(d)NO_cQMyU<|gJ*cic??KQt54%meV3)l3br z;X2GImJ}R?NzyC#`^1FgmHhOu)DjIc>O5&lp5EfTtQ$7CeQbuq6m%9htN|v-=jxDU zY_#pnTZ_#3BY0ezel?q*=;&QmiAebVrxVbOH9I6kgHmw~!0 z<%h4~IYj?w$vp6ao}zco|H)p;z!x;jDge3%<{W*3b{)DDRisC%pDdglID((tyQ^{V z!__-6P{qYLDn5hww1Va>)}~yU6V^&2 zx*k|0hMXvPvX3TS9Jpl3kWJ`0VpTR0e<06=sZV9mAfswjATST?6S{3+AXp|48K0+~ zX+MG9PPcOr$}zf4w+RwzX^j~wQ@3%AQtFQCcK>@Y68t{6f1>27eGDRj(53P8)3-$>ofLIPB-=F!oT6vGuL2Q7$Ml|~FjL>txZ^1l=>;gh` zZX7ec!WIW09wSI#Kc3?y^2Dgstm4V=*hEEt(eR~K6YME8#8T}FI zR|fzsE9VdU-hUqsDT4tp7I#i9nJqX1$qXrpk875W>&PgZcXZ4qS$~jb3c|rpI*-Cb zp&etJ@iwaM9Qt7gkOP?r204X#rcoyYt4qSbAM^D@!_WyJW9l6j_w+7Xl7>*>IoG3_ zow!}jd{pI??p`Va4nraW)LIb|3Cjg_5$rvbZV3J+@f0P>Iu5+kC^lF(Irf{jhsTw3 z50+y%te!OZV1=#Y&A{|=!vBxiCB6nte@KN${cHm~8Mj0(o5xCPq&?>(c z|G9;iV}nx`44~@jDvkW$qT=|pIpeW{Aey*PA0WuPl6_2}aBU!9lsME9ef&#BXG;-k z=7of#Pi`3IxadBLJKNp{kz}lqy}shnsX<$Uyu+rg!tUS$9ju1 zLS5{mcl$p@QOKdTFL$x&Tf%x+O#aKt(T2!Gk0iyw)nw8^LxV1n)m?c_A$WrURC}Mq zNu6C4fso#t&6|6we*+*#k0j3?0x2izpax}9!1ogVUIdm}>M+Nrlf z!WBP!6n;A=I|I6I73{EuIkN5aAP#v`^n*`vKeBCU6Wnz6SWuK??fXn6I$uDj?8>y| z_ALbG{+!d+s@yal25#Z`&rJEPGP7%>ct*mM{*8w7e#p)S4?^@6*2pGMr6P>nLpG5` zi4TXtDwW>k17ar*>4icLpuxbo#%K2swO)7Jal$vEj30~i?V(zimCy|CLym-71!S{9Dkec zhxltx>^(6MB>fOCK$T>D~1{oCy^ZV?geFTsDD8Zc%p225X%f|7+X$LUXtp;%zj*`(f) zA!JpZ`)E*E`|(CYk@}r-Z@PZSMCF4%at7wRlH-P?T@`O9Dm91`*glXg2qw!ZG%+hO zCxEe)Tks;DU-sS1nAx14H!EIEdL#$)d_>anD(feLpp5a*B-!)@i) z$NcR!2 z_nOq0NFf0p+|x`QR=858+-fLO@b_#mt_VHsw4_evzn$*NzkdRZi#U*uR3iqx zM-EB8ibrgKBn0({3J(d3pFgU!y>qaqSp*)#BX8N+||gssIln2X8X101t+N+ z<`lICr+`tFUGG9BpA}HcBxVlC4{3-Z!hSY*xFg z^Y?yj%7PJLXC$^0f48(nC0mCUDE{;amS@&0TS+pv#8bt7Yc(30CvD{Z_2KIG8E#*+ z_3N@(yh293xJm1vt~c=aW}73cq%&|-^p%2omQtQCHj30&i zx}$2E;4>MieQsK7rE1&SMfR_;vEAs2nk}_1PRfQE5rJg+$5V}1y+E#ap~oiD2xn%w}+(`&04Qxv|c2C zND8FXJ?-Q~folGw1Sm(*-yTklMgVRMC-E4Ux?03=(O)EkP-j#v|Fc~H_6cZZ3|qH;qM9~7H=BGnWQucld-`kCg=@@xpi0tmb###=)T{5^QN-h#-0 z4D2f|Ji)B}r$3j&J>+3GkLy4A))bAQ+b)VQi3DU`!x5L5i@34sEdfIHPOud^M`VL8 zo7?UO+CQZdm#bo0_9wEf(~I?8({RQF5}8P1sA7Mo9Me$PB@CkFliNo-VEHK*n0(hR4hyz4noJ__oI6g{~FN zXEE~8TuPbyc!v4lFJO`oj`|&RE1Ca>5wg@+3vtT60O78wVfKZQUD~gO$7QKDH}p=4 zjWmmoBrY=lQ3O&DeZ~JEFcITU#gY?Eqm1;&)5Y1>EvlvzeprPlNOP8>5+LQPEZ;nq z64bg*bvOOT&*QE~UC=J5YOX+>`4d-Ju?%+LXi6M@-qiWD z2HCF`d7~Qtnb*2u?`?!BwIyGq0&%<2PuT|N))_)tnX-p_KY*goV8h26io_M{FL8ez z2}kLHUujM)0Wgo6ofT}4*OQbF0Ca)DP7R9$Li!PQIZn~pAT2cGln3Eu8h#^x8(DqT z*7~f^B$h&pNhSr$L~qrf_kEL=>tK)ymA58IuuPN7K-&m^qsRV@;bwvSr)1mL4A5=I zG3;}DaB^}1GJD-Li14mF93ESMA0$@;zJgGC4Zu0l@;_*M$0%8veciWh+ugHm+qP}n zwr$&(?e5vOd$w&Gvu*qI`>wU_-fQ1A?m73vsWGZ5BQqi@Gvmpq`u{59iB9ZW9TzP$ zZ$CT%$TRfUA(D{X4>l9jbo6d|#keCKaMDm`p3}~~d4*mmxe11=@L^=LfCvtS32D+l z=yEuOs|eMHAn-A!Kn{S3Mn-hL@lD4L9VUGp0e**^mxMkl1d>CksHZ%@!C)uBO04K0 zbyucpRn~j>0PgX&W4tXdP6*{x@?Ly9yh2jHC9u`1_=EScFb*AOD?5msI;^-yCF&mO z_IFG0buY0nzED^)pZ3MPnrPFN{P)om;5x5$6pP?@#9AsjcDpJ*5$bDsUz!hL8NPm0 z3;qG+tSWjQQ9hn=<_3c0&lWnoz9p2?PCk%YuI&M=OkZ#$B;bJD6{>3qf3#(*8Ui|J8a(^9Rdw-IW0CKju;f?v`JL3U55farkDuL z$ePv)#1j{be;7%^6GvF(5)GR^c>kca;ufrIkV(7R*e>=R>|ag^W#L($A>@{ZuHu^? zPL2#px#e}njSr42y>G)Ig@shZvM-;Fu&%9PHe^&p7CUycaX@I<{J9>FN{%$llf9Xz z6XfCY5rJIEjKFb1oTd9mXA2h)NejZVGYWyZK4Vr00-ArWY#@KASje9p%@cw}h*X@A zD9#$VQ4nUA68a|4d^P3;n9>X-Gg3g3qmE?7QtN?Cid{Tw3sv**xR`4CFmmPvioaH?D8H6CB#56)^?617{lUGPf zxSTr7C_^NH1q4LmUCgJ8@$pm$7zr-mNumGigTQNA5L zw%QyIhV)lZyx{1j^ni3TLFjleH)dC?h?Ox;uEkf4^(C(-fhKWwvDkz@TnhH-NL^f= zcA@?1AXFjnvmc1Zl}ezOOb8Zzr$B+4!ZG0cAmaMbl`q>Z<|LbjZ*v7B0EGHqrF+5j zIYu26cN;2G*3a|e&67#jJU-?X#*5Va1yLfewN3WI05p~L>c#6ro?37lm6pQPkMSiA zMDL|dUx$=}sPq$Akj$QPfOb2rR$8}hGCD#n@&V9o%O-9pKdGgADU%zhS_&LcN9NFk zh({R0sZL`Q1oVQk0~VX+)9+L0=OXA%z57gkB7vzJ+wYd(YzjRm2v&mG710nMHN zD4UEjKGEoef^B%wDRFe9}K`^%d9iceqQ=f(T6-}zA^WoVb{?7?O4X$AVj zCrW5Kg&>D?H`%dS6pau_0J;h{XI8#i00jWK@Sb8Mznw=qLGOsq;LYWNGaqP4IOKZN zF&U&~3>kkyfbdMV%q*QiWSM*^xgv6;U~xWTHXp!-sFP0vA0!432tq9z0DS*&J8(Hp zSb(C;`HBBY1yKnV;4aJ%`#JIVC@(Z4p)fXjd-*&p=rOH(;?OM){m)Dwo>#XfotL&| zF8QDjNaBPIZxt)pdHe2RcH{iLh8F~%W{K0H4BZ8E2JI@42djz&bY}qybP8Z&Kmk+F2>o*= z2=gGBA6+wQ2v6$KCXQ%Tq2!AvKF2GVLHjpa(1qZUN$o_EUfCK;pSg2V+-{C91jBba zadNxhty*3b6*Yjk3mCF22bixQ4(-bd^u;b83;QzVJ{B;9u?#s_MND`LP_@Sz9nJOa zGLh#=O(HR zdGl?U&9{f`k|QC1{Sq)?=fp-Y3tm$A39vd1|p`od9f}r>&B4+Xjl6m}LK@ z=bWNr{hX6#Y)H*lWP%`Dnike|S^jaH2Pv_B&hYWUb`i~}zOWi|LEswLHcrapz!AK* zsAs(4T-LVqj{g`%2^iCZ$Lt3%E6Wce`GshAkCEiU8VgCXsy5Kj5l^NsBmM?V_WP;A zvZp+{+g%bW`}FSEoW?^Ux8X)q<2hZN^Kt5edk;hdbr1l6sKTAlTY6*%^A&QVLLPnC zS^-1Ya{N1p*qCkyk==L^N7w*A&ur#;@w5nda!zkMYx-mHvCpizWQsE8-^_;>>-ue3 zH&Y{P8lj5w^ij>iV%>^2d47JcSG+R%*ouIxkp=>@qTbYsffhjO4Q!ZC79c~0u*f;` z*rZyF!>ZQeD8ow8JU{1GI;7&nkHj4T@(0oC?#01T*Hf z{7ULzHQP?+qmF*88#7d4?`N>gX6G4X-NLLV$C+lGCs-@x8fD$qeZPIL)Lu7^GtPQv zTP^wgzJtOq%I5S-Z9UoFD#xL|XvjhqToyMEm7|9;mcx8Q`gchzPHWI9q=y9Hd(_=U zvKlhk3n`g*)RU9kcY{)O_X;=6P6Wz3S}nA?r! zkv8?SPN$^4f<3Ckml^uIf~rpxy^4agfBlV{=*WZp_QdnL(?z3zJU_Pv;(>wdGZ-F_7!gFB<``P@K-P z5fEHmuDdjn+#44HVM+*siXr&1qtEv2<~Ea* zSE-m4|FpIaFUMc+`4kC`w`_>uw9_KLshGn0b{!-r3lZmG!G5fAmpsNAp}eg>UJ`rH z;sqLob)1C}b$FRPE<2_j#$eYwl`igTWigoh_~D*JdqoFM=_U7_X83FO^cZ7z16aOt zmFcDBJz;Jc)Mi^;!E%-ynu9*K%ho;H*?_jCdWdRvIS5rgYKDCXap5fdgokzDG$J;J z+(|+q%k)eyTPu9~0+1gN>Hbgg>wov*`iFmt`nQ3KiIFoDosx^8^WTmtqIQnoeky;x z{+)vXicVfxM4k5Aqsv6rz{Z4toR-es!pMov%*58j(ZYz<-q@7lpLygB%uHlmYz)7> zy#7^8!q(JI*uuz}fRXJl^A<%YIzd4@cLFVD1{PXY4t4?tW)>#e@3@+cjhU96z^tWr6sGY6z_rkYF7z5jPU4I>j;$I2&@5EnG`QHf+0!DU* zzZ2i?VRZ72c1B7j&IDQnbl-kt1a!(K?#=|?760>)|E^ZThT*%+KlY6G`|{2`H1P5ksq33Ud8`7j26P>V)g|njvoq)Z) zuz|AyowS{if%SK5+St=+lRMkHYf~uOtN(Wy{+H(1|N8&`TXTQc{O{`gkMjE7IeG#{ zj=v5=@h=Cu|I*N368c*~bP_h-Di`{99erp0EBF7nhW`Bu`_AxRhIZ;)iHEE(6E09W z0Dr!m0PQXv{x~+$dIdcpk#%9Ro9OGj!PYuhi{EMxQJ*FrH`|tg9kL> zyDw*;lW8$WTxj3|OW85k&ilY3Y0StgW55KlX7qbgcetM&cqOWL&ar^gBc15$+xRVr z7X>G;93Tch)H^BU$9wJi->&#NO2`bQ5?A~U(^!>jjZS$M;|`raE|v16a&gu9B!dd7 zvf0R8whBCOy>6GlA-+ctLDfT0Hd-%vMGmUfmIEF^DtGt}`?ji^z=9z)X=pC%TNqTa zDKo*;7FF(aHMaq6osy^hDCr9?jzJo+hVY(c2WdoTBkBVg`o&4fU!B9?4_$4P1q%$n zu(ZdoUq3YBpJ;Jor8O=sDzP+19(ip|jT*)f@%&MN*S+uw(sX1g3+IRf1br}p;9kQ) z2F(D^a}7Z$+Z2f!M~29NHevvQ!}N1DwX6d!Qg{*aggKTIGgo8P3Ij-4KOj3JdM`Od z;c#bFm@-yK_yD7$m&V&-uhTgGpenc!T#i)I0iz*u`Qg3z$?TX%&6#3nPF|g$RP`DU zHj^0$r5svOkqb(ly7Qn*2~#THD_=_cI`LN*tX;x}*siP#GSMg7(<+Qfu>-w)1gN7e zV`KxGs7kOIauYk;Y=Y4_lGFQI>VdsQ5&PNHNaTohRmuRviM!6cUZh{_Ua9KPx^#=b zCT4)8u3+$<+cB!t7%B2_0Tt;{%Co1H`_b1Q(ndv8U@;8;7-Ax!$J3kGqq+CZJ)gH; zhPQWg|e~v{SWfTzO zprL}X>_|HnY~ap0tqOro%bw3uL$uHXuJF6hOWlYZ#sVS404uMNG zTNK1b=z>?2ibb$L!^o*u>t;bCX+Icvudu zDj(N9&ID#(haUg#|Mb|qyRgK)Jo@~!HRtY`jV0si>H&QMe`bC@6^}R2;bRtI!L!Y` zL3Uwjy!|*=A~)$aYgfFcz4P#NZkjq_;N`sggLWRC#`bv(j-<*v#j_Lohq6hlv1Xws zuDbITZEa_{d8SVM08fxZSt2tx(|;1uuu^i$ZU=X9SaLI;w;D)mbgB&1Kp3xdvzpfZ z)%FhVDTZXRwbau<>Ye)m#q5?iBU4Xc(rM4GG7sjMf~lKy zhl}ha49puqku)spXRNGg+O{DnC`%p%Q6?ZN z?M(?c^tx(p&Ik2yv1%HT=|ILc#Hd0m@f&MY*_>(KL%X-CH>8)`$~-RiNmCJabHJx+ ze*L!p%jPxThi!<$f9R(}gW`Ci9j88u3G^{nTE=r~lw#CN}^VXK)aXMJ{ixqH=>@zk~J7Tx4>9mQnxK;!AT z-)~Kqw-LGAwxWnR&cHSk67}C-Pv=~NcyjF{T8DSeNKgV>#|hoRVM-x@ zb{rnm@js9QHU0o&Re@)gv6sY-|IAaGKA1H6nUr4Ad!HYa&k}sR}U(cF>8j~dZOb*kT zlCyTweSwd2KEhgfg(qlNI0VSiEaEvCG&d2QXG-VP_Hyn^Eg>S3UkUQ&)!hRa5rZhQ zm~4Dzs4LZ&g&Ampf~5TZDp#Ai7v!226ZAsDIpu}F-LF1y1Un;eHn=kPeHtN0I$RuF zFl4sAWF@uf%*9IV`lj7$uPs+J%e$ePmlie-uN_>-moQ?gCF@LQU+cAocu7BsQLCd= z%mx(MU!j6D%b;o?Q{3Vv5OZXU#9XYmSr-b-dV#r;Y_6;RwnG@=0WxaVd0Aug*@ZFs zr)bETbGObF>miY~6mWmC)3){|2j(FuQLLOOfK+s5p|2;9by2K?L_D*=J#e^SC4H6T z46G!FL6DwjjYCIz;|>!y|0lL|wyzd)9Hvn-*QsI1rQ{3M(DM)f-kkYip92nvllfbV zWpYj;C1(s=-u%1QvSz=GM#NJ}Dt)YdMPU74Tvs&@4w=qd@{Vp@zkzJ-UfrkYcOpYG z#l)ZY;Ta0Gl$7*LlW1=2Wx1nuOy(3J%|#_TxzCJt+Wz#!(QcD2X)(6-Qf5Fhg6U8} z(M(qQ^-{(h@>cQh6*P-AFa58GToAyo6l#mBtR?`k}P3~C}2 zmCl7?4MIWfHw*!Ppa#goCLp0I{Ne4KQGOKcIATK(FVOzchke|_z(|>&_gFS$S8aA( z=VGQGUR_=8ei5Qo2_1rO*HO7K=%IR4MyF3Ec!i%_=(^9VV|Jw#X<&!XX(wV2hzJGW zm6BCiSbd~E^E2XflSO(|CAnEesiRc=QzR&gQj~ZMM0St@71_De0;g#^BI9xP%9f<+ zG06l51^;7SF;EA(yhvs1G9RZ@p}oehbe=q|pk?bdhW>ugwaa#MKWhy-)DF1VpuS7N zBZ9h(IJCYuhK61sddcX^Wd)o-`VVh%ZF@!6ul}G~PcvK>P-o3wGb#;8E{?nc2MY+u zz9rcr`?C$>s*5r6%u}6>zIltfZXQtCwCjxXUyi(A$ z+X0&Pq4$DX@Rn5=0#K!F&e?-@QB$ub-D|7Jv_~|4-sFx)n9q(U?dDiJhn?D}8mmY> zbmgw2ge^m=H^B(t?@>2f>HvvjqWAP~x;sF03QBlD5jJ|HakhFA54AowBvJ)g7}Pq2 zFGm-1AL*X4A-*9T{*d?#N@fnfCnAs*upkVy8&rG)0QlK^G2$`{eqfP^rq56G)sh-? zlP;R>(c*ya^xjGmN?$Q|dAU(zGT{#u#Y*k8s2fRK)W^2?b&JAMwX!a~!Dp(fn@HYn zdHTe0qzIq{FvJi9X2hG5AL^LRPeh)Ytt5=>rqf;lw}bE?6^0HF2vsIBg6@hy^pE?1 zHvb6)W0t)siw5WR4GgPx08RaY9Sj$Pu&WQ=q_!3Hm@5oKz9!j}iSX0qQdA3s^ax;W zuX_fKX$I|XHkYL-NTgN*=ME!)AT@u_v3?WYlew^t>?>uUM2PA!xUApenYGj~1yfke?{!%@_V`1N+i)6D=5zTD)^%|hDs z%?dSsT<-BcPhY*Dk2(q5`hGJ3o2Ao6{UlbTR-!{qCsEyrsDk-1-3NIdnA_QU56 z3ilZfhGl4;h^ta&!bfxCOifltYT3#y_)o{j-V(<|Py~0NnR@3w7~!-;nkhDTUtfOX zw7kvyCE%Bo^=Swb?t~eulHY(RQi*pxF8g--fwbuK zxQefp9L5!FZesvF+X?w-r%PLSXCsAzn@^6)s%K8krcSz4-L#_Kku_C`jiU&RioCm5 zUFBL3j*gkku2Pcy84`&7?pfIdI;(~fYRe@D$lDBZX5$eh$!}i20EKJ?$8aY@*?BZM zjDB>dti4VtuHUBk7Zm#EyT@&yeiRnT%Mkbt%@7&REa&R|1ZHY&a$JFIrSmg$ZBJGb2dd#{0iN7WdI8Pwr`Gd2G4f^aAvNVS z+#toknSIWsgU7OV)sl*bEhr{M$YSfU9$JI7q zg@cE$s_or4<)C`+6y|2QP8!TViAAEfUt~{h!#ojgqPlUyZ_Y-=+k^)O@pAS-rXP=MW4am$^9Ur-=)jt z=PJlrK}gN4fqES=u#-Q`m-O>lSbR2vYAJJ_u@elNympEP3P~ER`B4HHXAmdrNkX39 zpSQ?OSTb20&Of2N8wh5KcFspXr9QPPadmRTe@)ZG12TkMh`fY5To*cA&#@2Y4+4oP zTryf3?(Zw&PhO2){*M$62^M!>+t#6tTGFFDxQX&KpA|3aV4v>gAR=#!Oz ziT-am^*07(qW>FQ{sV(D(f<|a{e?ll=T%^${~M6fi83*LFaL{=GJkJi_y-@Q6a5YV znHc_6goWWd@t4V`BEi2=um24>W%|ai{~%?5mHJm~$^3snPJI%`ZTtCQM4n!t=nN2z zyoT;p3cKA>>Wg%Qne!z<$nU=vC71$c<|K(ccF&wXFalW+Hqh^hd{MFq0fTCXcTiR^ zM9|>Jo6~ukVc0tM+S1hq&trjWMT~Ozn*v7=X}t7ff|oSPw)7cR<@72`iCv5+nA(kR zX;v22|D=ofQd7o|z#Q~}9Uyo$eE^qk!SwF`@!>!q0YVmy+(7QiM-83hQXU;?=D<61 znMd_6ghd-;mKgc4b6YpbqtciGO(Ul@ak}Z|U9h=Z#i=h~FcMOY6U7ndH*y>lqEim^ zpLMa)45$;wDmbHDpYl?v=cx2P^}%oys5#LQaM7u7W-1eKywSrMM!tfg43JeDGgHPp z#O1UD?}J9vVTd5F3JRU((YqXt6k#}_9TRudOnyL8(vOr4>(bHfHbz!J{^=Yj>SV(5 z;GkI35vYgn9p0jH0}4y>>+%0B{(rR5f8exiEGz^}jDN}P|D4k@u`zM{U*NP%On(dU z|0k#YzI^_vzW;liR-4Q5ycKoQ3$+U<^BEZX`XH4lji~J$vK^UhtNSVsAWP;p;sK?c z6ayEKyO)Q+o-cn(?~`+T)BEeK$L{O-$fx`3j8%?h^UE#k>Un$nv$YMaH1(2q$insM zh^Cc$vFzEP^_haWcTMjmuO-fHVJn3qtV7Pl5w+Zm@38P05Z-b5>(gxUzS9GvxVOk7 zS3C4qq*5-otmWYS(M7dSS_WTivXjMFn^Nq}zR0M;iPG2$>*5T>u)7k+Ma3fso67d; z5p?X0djaQ<&E@jAMyDp}0hE%8)JppyGDmNo%zK|ve&7OODY!;9+XWDbg$*@gZxFDsh2=5 zY;yR78Or?!go!k*&0K{5nyAmkz#u&!@wI11umyC%P9dUv#ljHAKWXv-I@&ERv5qJ} zyb#=khM{K6u;=D0j-AnneL#OJtY1F*M+GcgWV;FE!wY^0bUUJ3zY%HXTn6x&B%3pew|Bcf0(?XaxtaZ9 zy#9BUl)9#wv(2lqthzcRv!0?#6% zYU1*pDzX5FS3q=_+&5Z-K?7N+)tN3j&C)WqHQ7Sz|Zf{)IhOQFMMWMtaC;$dlM#AeWcyX%q9AF?Zwn*v3eS`;(POXo)S|ubFIA4Xb9pn~}0z zBMZSpKkL`C8I!2ypuDfVGQH}o_z=p6F>hCUN5I^)p!b#Fy%zwSN@lENk3KVsyfdj_ z$b4ZMgnc-vzKr`wSu2Zdlk*&~s}2>8%XpA(2UPXdoS?9iHmUK*E0U?ygM_MphYAs9 z^8NftZ6l`!s;*3eDnk-J3T(78O5zU@o($QVi%+;`jZi!7(5G2%m!z=jbtEu`9bGKr zGnpHqy1ey;nH+Jy-@-ZMYJKEKGMeGDCa0BiAx?GSEWrp!pjRA4rEv5o!Ki2c#QJ7V z(LA)?K{5Nk;0 z@}FXZ3c)4Y1=K6c0}nU>?8pQQQzjbbTMUS%%Yno&E;=jrrTq}dW_6P#+a>LEF}lVS}hUzmy1qZ`y}g#5GWnxOmo8b}c#JsTxqQ<-tM`|DThayj3YQjjYy6UZBR z5--I{GqfAN+jJJXAX{$m1p5&N>;7XMWcw7NU+uAGt?05!(04QZ@iF;PUNA-M))-0b9_X_bkkjl8WXuGwQU=_-+@fabn zY*sWU!JntbaKUe%amb*x&T~>pW?E zL!V~4_0S6hm3zk}7dS;+M zfHc+SbA@s)sJl{shAUW+{#M@?_Kn%9U>z|h`DTDFZ4YV3O^pKgn#%xx{n^ow7$%MTP^`$c6fAB~%5?m_*eA^K+eL`c(>6_w4`Z zqEe!d>`nYq546&LuESyq@?^$}ZyV5~Ag0iY2g5MfUlI#amlH#z zE&W18BIsC~-0>-N52u>S&cn!5qZCw1E;<=Foa?Zz^V{?z(7bf#6OhdJ=Retf0dwbq{O zXBiJ5Au#zwFXSvZTRc24Aylh)mo5?`qyEz#gnaYSW~dW(?@-v_J<^m;y+hb*NRali zETq7M+t8-~>G&CQekNFlqU4dq$mLI8ajNuu0kgYIliFm5w9lWAfap@gTt)RUW;5e; zMi=Y;dex$;0wA8Q+G@oEJ*X0)IP{9H>}SS@^=v+_DhCNq=tq^R4+v>*611|~e11o* z5arBuplod|v5`=EAbn5#x$ynKam6^ix`9C{q2H9JGk}6pt&-N$F*23pv-rw z={FpvM<}XbOI01W%c#Mxc?Lo$t7@@~5gehTs+hql@sRX0 z9*n1cc+T7ftyanf2!o(?>Tqz7*BVgVvJxTAA=#H`BZwbmF2QO5@AwT3XYm1;;Us~t zd8_(oLxvD2Il=#^u_-+L_X04rR8G@?;Q>cgjJll=b-I9Lk5-hmQaDUIe>BS> z(Tllg6Zaz4&Zo)lZfVWg9E$wy*C`w8Kp#M{T&hNLr{d|Cw#FRtTfa{Ei^0vba=%r6 zUmZ5s1JZ0z8r*Iy-Z^R{oznJxbU~{MWH8^HS2b7^^ahiB4~MX{aP{0LoacGmZ&&WT zG!2Z!x&ZVtztvv#XHC=J{I;HRUPAcsB;1|CrW@pdejnezqzrbJ%eS%Fg}09Fk45Kc z@wkA8NsqCxNg#XqL#%P&lsF(Tpg$reBQ2NDbc%iL2&i-7a9upTcsN;isR@^m9Bxix z(dyWHL>nYauf%GU@G;31imLNBse69q>j)NA#nKh)6lo~8;|Jn!hl>1RVmtc&II@m} zDBRO{r^297oO(l--nFk_p`~A^@hOtH^@caSYac5=q~x4N1dZV|A{GIu?jXae7QVM? zesBHG+~YPtYtHbsP-OE;}UI>29$vVFq)b);*R2)e|+exl5r zZ`ZWZu`B_YCrql)Yq;t_ov1W>FWOkXGSEQ3Uy`rreIsX>_Z4(T5f3G2X@QT`e#G4_ zq1l%oP(0%bPm8LXRpE~5zmxb&3jxU7DD@!4M?v~R@%?ZbONaIpHU2rfV;|-({ReQ> z=$=yy%p7`BZ*7F$Bix2na7hRZx!4`6E3$)QpJ4iqb>hImH^Won5%28n#^vjF!Z9GD zot9${-aRJrrl5bIPgJhfP1h&kx(oNTHOG6QuO8Ajy#g{WWVX5cP(mu^NHL#JWXwHn z28}uXeAO-Benzwe1$Zz!NQ_x^p)@tz{#?rw^*YjC`R>(JbY*{WTKQBX z;A1SP!b4P74G+wp*u?t(}49kJ_=NBF z39TwUu$xHp=R)<(&J+i*PomyC%XQqc>x?JyDvdv<$tsTZ*Yw{4iDl} zS76w3%!%MSI*?l*-CT!yJGhTIo{RMic{ct?QPF*YEeb_j=iuxV#Pc0=?8N_GZ|{ml zkteIg`at74vU&_;a-VSceEg1e=9NY1qi9eCW3agF_h*4cKbU9V6^>~;hS*-e+O?K! zmFWcNWVmlTi-r?%L5awuT~ennM#$J45Q~~yczp}9hlc_e$NW*cTlmcZL6YP(GwMqM zEeJj&fY-Ioj!R8GN32F*J8(Ok^nD}$)T}IaLlCz|k_G)nt^70aCV>5>t3n&1fFK6cxWO)a+b7CT=?X2rz=|!o@ zYWdamq5?lGRB&y!Pv-YJQrlQ~AN^G?N z{MgoT^59&6{x|?|uC4@>u>5ngqw9lfb7Qdl@P^OWAm*zXnOf^w9RQ6u7dSjs0;&K% z;NjsJ;MLqz(cKW2_+(D?>3b)kU}|fdfq{urRGbf!jREc%B{KmqYk^k=7vS>uY)#Ez zfymW^df|TQgZYOPLHAzffdDu<)33 zZa<>mUnUab?4B?603I&@)Y<&L0P#s_xV_VZhhKi9pluzk{ zNZQ5*$E2?uuyu7R4l1?twm)@W)a;L2(Zt^7>S`%hftT5*!SQ#fcM!?TWNv;_ytk+s z0njqkx#D_Z6CbS8O-&ovF9v>+EeUjAq_{_?AwL^yu$ipRCuG1g_8qy@>j(i)Kzy)f zKTff5jyvbFWbzu)Vl(V~2E>FTO6zDlddbd7aj3+LT% z$AMV1r9NR*6k6-*x*r4$nEEQq@5ibtK!Z_vE1bj0bouwE29f(AMa_4f(GGwOK|JE1 z?_@EQ5Vy#ST5}}q3wn$WDkDl~6ib>8v?Lj;d)W)2zCC>0>wr(>jRs=)CaXJM+7T}# z2%J0Iw}8po%-J~R>PQp8qYzvM==faad0ffAHm|dmisor+SUCsp?kiDUTK}PsH(h(F zK^|zZ7c5l<@i1X?aFSxu3!Zwp&I4QarJ9cv(3vNVG2sNp+}l8<^7ZvG;dWKcIP*3^ zC6|P(jLltdm|DS#tma#3cdX()p#SM{h%Ky!h&QDWfs8Rgjizi1 zV~RJrwHV@koy3Bz9X(}YErk-AMd)OSCP;CwuU7k5Q{1kulP;|9>S)+l% zO@cs-qQtL;OasBPc(+W^q4_1e5;oY8+UehS%GU4W)ckR z-3va*(_Gb!V;F*A1HEYU-kqV$CoR@vo4@#u&?Fr+rr~C8lixFlgOtw!$!_}Y9%T!w zf^Itl6IlGsmo|jN#RHbBuq`{{0W3W%_S=$>yi3m(_e*xtg8~_WN9LGifo1Nr^uECp zsgr#`h7{VGRW7^NyVTi364e3?8dHj$IG3qkWpV22%Yi!xKF1>3s?aa!9;ASPazUG+ zknlKFHq+@uO3S+!?+ndThYbBUIC#;IIm{^H+^F;U_K7%f+7!lN$BrHh`%DzLhkeyk zV9y@E%G1>E`F)1R_7r&d`mjiB-kcLc4Ci?g6m+pyqws9K)1>Myy|Qp@%6e6_4dX+g zPN4BUv4DNNMgh`t^Y7_8I}I*dyG*pYLDR06N_~o@1%i2O(ukPyl|EDz=a2Y#2c|sC ze4*^zJtMaY$L@()3S}_OOoV$Q9-!wGJXHqFP{|_aYNJmMmL4JXyb~0<>xaR8F1(Rx z4St&(EPzI1epodK@Ju1hzH@D9Gt|8GxHhSY=4&2SJ~~NW*$rSaW7<&pyT4A5z&Hgi3k3h`i})?KoXHD~uYt7>hk@d^Ff`Z}v`m zlh6_f`rI+_+}eQ7kgk>(Egf+}{NTG*eL!tsnD5!9^BeF8t-_{wcq84NW~4%B%mR<4 z__)l*1X2vrNI@b+rT<9s)@np@v_g(D1fZpLwy*so*x6zNHRM2j?zwti1ba&^m zNSQrW)k?l=uMiFCp9VPm<$?51gtY*6mSMd}WXW?rOjSXo{imZe&-6+!45HoAdugMC z3o9XgJ6w>Q6Ifu05t!l0BS4D)v!Dd}T3Swx{Fl(<$G>FaerukZH+xnlR$C=jcaR%q z4MSPF_lEi)=fGZ?KQZ{dY|LcYht8URvG`C?t)y>k+7-I-XOrR4_12K0qpA8kimPB9OiqgWoNuW*uaN$9K180GrcgENSQ{LsP;a zQSuR}V8O&6`#wvAYae2Ij_Qg7!-8p?ZhS5w6EGH|yngW-&(p)!CuUlSa65%XoI4g8 zwob>07oKOnzU2)Kr8MG4Q0H+mSIK9sE-Tz=zx{Z%TgCI7!u5`n*ax61pS{w@aNK1G zgK_5-M@bSw#b#lwI*lbQUz28@I9G3+hT)7tk)?s6V5RC%{ikhDO>YWJ@?CU5uAHmf z`QlB~_Hb}T=Dw}|^H9b^jHSmNIIucqgeaW2Qnlsb<9eJEyWV}UnClg7I2g11AOH#hu^r0ca-;$hkHVsICF zZQb(`ACe^rJSFrR4jfr?8REj^2GTxD%pkcs%*U7^^}fT7po}XTZ)0XX!xwt!E$wVj z0jQ&t62{HUyr!_6DAFq^qVX39<;MEC-UZTF2?#kFNyT-D0y`t((Ad=^ddry~=8YEs z-bo9xtG$~^-W;(&kL7{<4%%EBJ=vY+u%FC^ypLfQ%I2Z9sDZVM=BWI|ebRfVN9m6t z6u+(JWp+V|N;7hek(}=0Wm;_pl(on40ieEWP1WKWswK(0_M+*tBaaHEY6vJ4I+K-y z6`~Gz+xq z4kR{}#;BTO#``i6`T2{Q35i0FL}E=DUQ-3fs4|X89%sdQbg(4Q-y}ThT~iq=Oh<@< z5OF$7$u_9ncv+MRo0BusD2b1agjNyw9wZ#9er@YeaF%GLtWt1hUb@Z$*H_Y2;5y!W zGLPmUKg;^kKQ>m=r_hiBt)G2Hr-jWLi@W()F@|=08k4=YI?`~TVx+!wa+4W}RS5iG zr0))1^z9=(#{+eURA!r>jjDSdt&0LID|@evDw=u8LOzyL0Toc4iWN?BiG8riK!Jy~sk=@9lDhPUtp-=!!@iWv7u^;hph**TmcSySycJ&?=*#!VvFt{e4K zyp&c!2xNBrFd{aiK?Q{xFZQ^iw}~VN55`DkSjQpi?#tS4_hH?ZOA1Z!zH)Oe1V|kR z68CIj@#k#FZbjYYrYU!(~2J9Ki4eM|$z9Pt&IkhFFn=WO6o@94Mm(C%Ys= zdolTdz)3_!AXIVAXXQI1x+^aX&)~iBr50bKGSN>mDe;aZO)OlWA&R@c(p4Td%Kn@vmbzQJjwej!qy(ll z5==!V?55kp{~15aUns+ik{r~E2_*S@18d<3V%>fg36>!<*UImES|SpQiX`b4J}bfQ zhLi!(k|82~TQVBd$zVY3U^L&x*M16BWn{XUtK(dMBRKIG^Gt8)(XplWE*N48Pr;nk zoAQ8HsbYK(oe9@zaHqDn*s8J!gH^lJM1BYznda3?5iV zt5Z^HH_PA-;mM<~sr-8u1QE)6k$nqJ3|WFG?*9~)X!jAc?u`B(d54S8x+%^h>U``~ z_vJ58{33qO$aF}L&jg273afb+5V&F|KY4N5%u3u_@0^v z^NORTt@dgMv$xaoXV~3~Dtz)|DsxRWj_|pvQY6=MV(pp#TVKX)rQ{e-pO59HH(ayn z2Qau%L0U~1l@&sc&o;w&0qQC^H7yi673te4NB>p*uj8Hdsz4-3bSu0NL?9=?Zi+0O zeo@O8$_S^h#v1XO7yLKVNI5C$ca95aoZ@2Ynk`eNoIZ6vZGU79xOF=^v!aTTKUP=U z4nz~5`htJLx;73PPCyXY!-)JYKP!a8yEGlkNU^2vRjoKeB5xd|%o-P?b$^Bjx1g4+ zsR*>R{#>h`&tqZ1kR}U;>H`eEueh!6sh`kDu z(;*&p!mX#PBrY`7Q&Q3;RxKb~`xDp)?G7D#pGh@Rb>f?9vMs{8a~$=;CEj%*z&@Wq zpN6u-vXi0U+^gj3C|Kgk{9d%~IquI)->-3*Gofu1IYPbRh4erl@0_0Vm7BOnmV<2I znRr9@N7QHts`S`2QHRqJ_#hHPG6{6LQ-=i_gL#gkqK(f;x1TyzNOIWOtQFOq4#kO} zGf@3Tl4Q}!@pLVO{@I;Fc9wz30!MQa)Ex3-h(#E96+>!yFHRhLF0V*_$3TWiPJXo_ z*fxfZ4%O<`I$qO7_?QxebDF+vHn^gbF|lD^ac)LEG4jOOD4l&rLh|@Ck0m(hcsP8Z zI1*yKXJal`glk zJ!Ai%P`|#a_u$L~pYg!Cau-iMsfo5aA~Ren_DucY-b3u4GnDFa)-Uqa_(=y++@u3@ zl?dDXkw2+O&Z6qI;aCCVYX{RLv#j}9KlKBzb*16_?UR)Urcwe{;zFRNa z{bFqEPpL_43*lqixZGu8dGxJm4mLlGzS25wO1yZ@ZF~Y8<@CI&qZ)A~5#j~5=7EOO zK7dV6$0KaGi%|TbB^H2OY#?Y8>ob1Ql(ByyF63nKga(4oDbyl2Q1`zWJBKD=m@qJ| zZQHhO+qP|c-?eSqwr$(CZRg7&IV3rx<~PhtO+DQ`LKCX(QhLG-ffokTTH1v1!<_1{ zCh-GS?7gBrfrS@eAmYvGOOr_YqG;KOK;pJoqmT8iFl86J2;Qx) zFWJs`ULDtXxKfBkgUw%d8?Ht7TD7yEjw)y-EZx}z@#8hw|2p7IjBRgmx32vm8g*WV z1#&%h0%5Vj=T|Y!Xbw_sh)nGp!d8;TTWfL*9~z|z%6JQvc|1?mJs{vAzsT5E$OgOQ zWtV%adeALKRW`y^`Lz_$d_>>K#}=7P-x6!tL8jeqz#KUjr)Xx1H9oB5DAp>6#{CXG zSk2PpR*V>#jVg1(gr2W=(eLiSd~$AeFYxxeyP+SVgBYS+5+nM{yEr8w)pH;si(lg+ zFYj;U1G$6}Fq~r&&Sr+vHj2*dmAmJ36DQZZHX&w4WOyl(3P0@rC-M7_W7YH`5e2@ z3F+Ag6`9~Qg46Gp`epOtdl1Y!y?LH^b>#tzfSE7bT>W3=iP!Rx)C^31AO5Ahy>IFanmOnBfI*`6O{-L5W=Q@ zTu|er@MI0C)m5lsm~(s{^>dnwy;j(s;KZVW><|MNMB-8{bO8_S`B$?fh=*5@_hKys zB^d*(Yiru?s)Z{wS~{Dpf-s2#Kvr)UUa%ET?XCDXe41p5%=+CfUB+!r2w8hX+nx^F zPlp%#K&rBSsuN%+GL;@Uzq6dqa{nYcy7?o*Eq2Hb`x7$#TcPV8Bm< zy^FC(BGi79uGBE?qO5k`S6&f4`Ddw5X4+c`O^$~?(S^)RCL>~gA*#gej1|c7;*x@D zQ`K$CbIW-M6Pm^(@sf0@>y8EjMA~AjKlip5O#Ur45?<5)`QzFEai0sMcn4Z=O@>tV zM4Mfd*zuc)*VW|w`UnSS11`f_^B+w3YI zCQRIvkcL0_H<&WJp93XC6?8%Y5&>&?4F9+Z>Czw%{B>}~m%Mwk5y|k@V_%xMmnvjD zkI~(IWWuW4E2heXE`Re2*9G(K?;{3|?|_3jiptHIo%LFw&-lU|pck`bXp z+dBH!3yY^`CMV_PW9x`YhW6`Lx;L0K`Rr`hl^VYqE6W#Xndh+ig69bLl6;2<=U0ET+3ZKCB9DD_5=~&Iwh@094Ztj6~*GB_g~g<(Vf-^Jk>s zqXcO0q=1U}s*=p51omhw9n0OOrvCHCwSFA0Cn?nBKI&YZf22FVdCo0FE0#~^7 ztvsGgK>$UguCjWkwhmS-IE#h!Xn5Fl8wTqG1IG?*ivKvo#evrA-fxR+m@sK3e#NZl z&NO$|;3}90zjOlW8V@tdE}sZst`+c z$hb(x%5D0f*8z9Bt_vF}o(iinqfwb~P;^@WQBcDOyrjdM)K&&ZA&(4?`#pHDGI^N9 z#yIDtfxn4C^^(l+FsuD5j<>b$J;Vk1?li*YMaYxJDu^RbdJo8prEi^|;3fq9>=Zh- zF_p0r$hLzE`{v4JPM4(3iUvIn&7!7dh6a{W z+cR9?HCHV7ab=F3cK(1)5iJQp#Y!2LD>jY(LxxMn5k*m4N7#|*(?#*Td)jw;$9ZRO zR|mgF{b&7VxXW1>RO}o!BJP54qU{PST|AE^@87k#k=(+<&SM&So|M1=Vp<#03t=jq7ICQ znscs=%Pp3IqDk4Maj>A*9wgm=q$a#JyAaY1;0x? zaW?Tc8Y=TdtOfF-O3p7nr&YhjIx@|9Z=qkeK$!0VBC(Wjnk!pLM7Wo!{zq zP@0trw=UFL1`LrW!03$Y^yd=U2qq<)T*kzOusukHHh^@QJzW@!!Ax&^z7o1r{;z~c z$aJ0#Te0ZGm}RcvM&#Y2_G1*jxYLRSvG?HjS_jBVa@y3FRwQ2K-(XU$_M12Jv(?9Z z4^MhxjYUy8imMpSgaDH@C{b>CHH~HHmU}*YGP<`RnK~9hoSKZb;1XTIk z%NCW@#R{xsF9pfI@nAcQk7=uT;)g54eB1?#pg8eYrFQR;uMvyS9;BvLB5 zc|<1nd-I6^g(VmF7)}pQXi=PL3$3NYb5vGDZszRm!sFs%mM&%-F)o z;2re~NS-_EJil|QTrxBrgHU?T?!ctF7dE&&(n^%Ali9xfb=)ucK~s3oLL|J$BT0M{ z_HK`a?t!t@aTmbxMF=0TZ;91=__>AjmH$q(9}9V-K!|lc1Dx?nA{Bf@7ZzucxvB0<;cMkPlHM4XcQr) zHj`unE0O_ca9XXZGY3ACfY^e1U;GFJxm_r!EUcmdqhoN))Wpmx#3Bjb;_&jrhQ^+z zJL^p0VEIRzF`~K2VH%Pv<&GP}kXZ9*{IDvY_2pY)a`^p7T)Q=ccAfS|KhiDh%F$xC zXu@Ih3c&sXV_)z~7jYC1EVp|-%BpEuV%A(XzHstUWP2P_J&X~7D%-7qim3FTW@Hnl;|EmnjBNPPWzZNjTunzxch6#CDfPBL>Wba|m2l`V|(IiR# zk%i*Q%;fb~K8qd#fmy84ulAYPcF~YrJkJh`!Xd7Br@bZ(GND>AcQq-htM$*ii1-S1 zypz4}fO~rhjypJ5mZq6;Fxz!tt%A}NfW8JPiANtrDn;$$$u-(Xvy~L3`h6E&bq44y;Mqo0JW+epK#6yKdya3Q!N1_R2jm&G&=V+|x>!zr%Q1lCqcXG=Oev5cAOa>>Q zIB2|^Av#)^;^+3Vrd2-^6sLMH`LecGI;xCPm!jA|m~5?QH19lC>rg$oUEg;g<|tAE zW6MF(bnkM{TqHCkOlRb6>ktRJuhOM9rlosxWdXYVr2eB3dEYU(p;df|i3KTRo`5;~ zS@I12XwR9%4SE~<^fvLN@tC6=gMhwye1Q`O4J%l_@MFC7RKc)BBG&CfZe4&Fgmm<~ zy854ncKo7ddh%5v&Y#dXm7{QYI2eAV+WpzjuN9PQc9&?)bpd!}vN~h1k_QJ?3dm)s zk*A>!hv>$KoT>;BHv6gm?PH1@iQ}B`Vp}Vgg9H@*QuB-teBzRDoQ>yLD^HpK+HTor zy8QBa#KFq+zsQ*X1DauEVq*Q@K+XR}&2$>OnC&33Vr{wQuD9B1 z<7&%S$J%mVthe5{*!<6R)_azHmwmU(c)RXzraFUVy{g0d`rAY)L4y0Q_EInQlm><- zdb%5cRZB6^ zR&RDSRjn4lN;%V?95B;u)NJ*2brwJtoEzNTTA3Q4udA!8zAFm5s40u!h+o#~^b|my z+WB8dOAe=ijHITxm?D$_ZAm#`JQH{(H})1_@{Y`PwJhM{YU!x}x--_lkRay%TYkX) zWk3tFA9`83=sz?77j}2IAMx~%rXXGcK_OicB}qkG0>aT^Ixq&XOu*xkKTGh=&imgX zHg?z6zcrUdh5lzP(|^wN zP^V_AcCUXI?0;i%{&vgLfEH&j{;}WY(f;`3B=mJOHDsc1eW35|nm`tp24=9$Amtsu zMvsEn)QX=N3cDXb>{O;)nSwU+H zR1HP*j||&qdBI(|=Vq0_hhkVfohX2@Nb1l<@DYURNQrPWPy)!<2?m!31My~XVJ9>r zZ$&xEkF26kF^BEFivPp&fljuW$}Ul`fkhU4a=zr%=633mX;!T3r<@>Vl$m@{tqh2^ z-ZTn|PT)AMN*b>B7qz{&+Y5Eaw3!7P4J5n;yv*Az*qu4SR=-6OItizHLvdKibuo}3};?WNA2}6<%utWe{7TOLchJU?Zu2!FHaI&N=Y3^GwgFV5WSZ8qBJOGm7&SH8Et3Mv}T1Etcd_6FX5&<$S* z7FWo_0E7FYhmH(cfp+s7Qew|det1!SVaZsE!gZ^ojdF`K)7=pqHkHtW_^|~d z&NRBDIJit>T|W$7zuW&I`KXpo!`(5>LGMAIBG_3;Tq&%B$J^E4KHVOm`Pfr81UA3rJzh)CT)#w^tx~dk+~_yxlf7BqHG}cC8T@xY5BYjO2_gvnjCuss zuF_qtNc*_Md-@ooHrg&xctK&I*^uMY7N6=WK)YEFPI`C8uZ^PLzJ>EQ0tIf(4CUp88xL zn0zqW6%?jKKyC?_kbsm8LPAQ8?8z*WZ+LKbLPn?KXH*?$rdB6VIyLAl_zi&4e~i#m z^ID3bJIyBKn$U}&fQaSX_XefFtn0gXr!8$=%(3-0DbPJYzRY1O;(>p1OHCbD#YsSerAQ({PyrbXQFllRc@8ZD1p79X?8p~X)&1L$ z8~A3#QuD2MyNG>6QP+x{!&*xe8;|p;XW(cu{lu3p{?3ykR61cIMG9b|prOf%jgNVL ziM$T-K+$%es%Wjqih@jd856$dn}+7xJlXm@-v>Gs?~#Qbn_FUR0SBcA0ph;(WYYJa z-6Eh^Fpr~v{djvBn?-$-u)-GiYg=9rcBXROJ2iTE^4>$@BU@Pmhh$IeXl?y_Ds8`# ze&NDH)L!=mo%khxR!Wn=UaOyL7zQE9)%)K;1ap&&N3=?@t;a)!U?uHqiTM!jVGU?? zJOMB%W&XX#qJl)6Lo)0g9%I^46^ePUz`|4Phk96+Vn(0J-$hvhJR<#$4~IJlSalSz0x@8c$GW_6hw*&y<*?nZKESsJ0`84k)H<9{Eo zWveoW;1+kxtyZcdjGf)>6*4)8jK>-`3Cb-$E1Pio2&xaxaok;Zsx3l9{rzA6PqY@h z_@ZLG1QuTNK*PHx%T0Q2-LxeoT)(kuuW;mqW?{#(V{H6+=09twi3{N)WQTv9Q#bPP zk_p4m%WfsHXx`Dq1RspK&Y_M`rNOs2EUu?v?bWNhuRh7;3XeP~bwhQcQIx1B`pPFLQl`H@4(WqsvRV)><}=W)Exdbp)>k zcOyP8w7xs#&P~bA;8`?X1FA91Gq9dYH@lBs%2J!r^tjsYfLAAI)basvcoBOJuB;z$ zWh!DP3QCZ2EVBvhd{NyA&5dvT(J%=aM9ClMCTx%Sqo6?;NKLi%K;iZpY=sGO^G<~SP!$|}G z4$B8^LBr*)8q2#aU4Lbi@Nc9`TO+(ftZq{)3wLU8H-ZLK%u#Q`p{CR@A;;1r;+;p) zmviU}YVZRu-$TE2Calzz{O7N8Vz!2uq^W>+V68N}~5D z0}ZD^kY7NT+3VXDR7~1zJ1qC&8=+=owJSV4ECjwl$I}Bp*>ff3?q(|B<; zXN=sFN*jy+A68r4_2Xyoz7z0wND|H`I&X$xy6({S!Dx3u1ZX{ z5G`fzj$Ie_;!~@K^m_ssxxA@w^x=0{<2O`8cu_O=CF_Y|ll88tx-r|pLhHzp^zQ{- zrD?M~cPw*ry#IvXFdLv+J%UN}h5XH7G)?9E4d$(Rm{pnkxG_y{-cy97jL$EIGw0D# ziVmuWHM`q^X3M87xR5KVHm}kc4*MdkvFiD30te69>`E8}-I$M1`fPxemPh5SmVVAa zD^Lr3f7dyubHDfX@2=EE*3i+SL~Q$?m_8Fq=BN>i;8K7?wcZv@hFup3*hDF z3y&B=lm_c_%g6lNA!EBZk#p=_qL`UB8ceGCfk+#wOO3bBWK^R1`0m%{UW}*#8j@*_T6vSn z3UK0ok3HSi(Xf_q-PbdRj0;h)u?Q<4+_>PES|d(%R;%h?^ybLw-Ik*W-(6@Mn-++w zRZ~eqG>e+4IZP`G8}nby!c9S4KKvS(bw5VoO{fosO)>dUqgJ8~tSOpN&vwDfQ zAMAeHwY&_FXxj8AKeu`lUt!v>Hzcv(b`W~X`B8Xb_c%^c>+iR%ikGExB*)+Bb~I}t z1*4&r3b_NtsD^ijYc}&a?$##ynjN2Ir8KJD_~lX?m9(rGIxf_&HuA4H%# zt(+1#?MG22TdYu+wx?L50EtZ~rEU*JvYSgZX$jTLv%>00vXIaO1|E7G`IWMhL+XLe zeFEj=OVUIpvYJ@W6UE@h2(54L^_jeRBELv||{=C4efcEdVngHEzJDArXd zha4LBTbb0Ru5uWuSbi_VVYqxSgC;fO^*E_q9<}(+U#*nU#{~Ps5Q8Ysegy`-Gep(@ z{Cf19-|JLM$_mfeol}6VPbr$0Ko{XvwkfRXIkZwvh7Vjy$U)AwJNb&`2sa1pr-b7R zJY73muhRYX<56@f^wiP^=38 zhUVT}pKC|EoGWJ}d<6Awp7r_iH$@~0p?LVwlu4Dul$2ydA%5+>_(6r9C_vL<`70?~uRu4Vdc|Uz{ zAX%C@{MC22`Lja)Zlv}zayoh~#NwIQHeP-;4S`fjs&r=R!8y<=c48TMbVOgW8=e;v z1IUIzhXh^fl9Vv-q8gUlu25cdIR>7f{$QIGx#;{nyFpEv6pm3omf5O}PUr6(RKs_! zB8=pZF?!YC+G^_@f;1iH4-YAC@eS2TohSSn8+J@KxYjF7)C7gU9tv$gT0dF(hVV3W zn1#u&1)v5yXr=qWnFQ|h`buE9j3wyOb@?MKqg=(+QYdXoS{q}|EA`Q0l|%Du8xG9E zPGvJl<*qoK+)ajrbd$~nmO&Pi{%w}Yh!FFk2(-R7vPsw}KXp58#D>9=R(Y@K2MmJ~ zXnkHq0)-g?5{c^TL|7huQiy}8PP~8{O7zer($Sv!r4Q1qH6`saloeI_#YWtVeR>L3 z*v^`_Z=bT54`bPR^%sR_oY`$hSN0YlowJ$~#1)bn-uWB*q45h~mH1XdV?)^L{zMX3 zrdQYqit&4JZpMH;^dKi}+@(VJToWEe&tLCyf6RAXf-NKbHd#w_5)@rUb;8!fpz~}s zBN)>&ktgaZtt9s8T7ji=$n3qpFVktyKslLVAJ=X~IaGzJTkS*>U{cb^fq<17%=Rpd zh0Y5WI&-l%I{OUCa*#x%#57@~a}R1KTU+7p~z}m>@U#q8Nn94@z*f zhegjV)osi2&@5HUe;KIVasOTh*nVvd$_>z+3=t4CG=5!s0#1e-g;4o%J zwe*8A?4|q+WB1p!X{V(aVQ5O(p4;C5L z79E}Yf>9sCl#QeCiQKS1Y?;0_6AC(!*Ap5(xG-tWlLs^}bLvXbo88&!>Pc z=l4U-%*IllPBiEpCE2))b8BZPoEi^VTg8*;lX2Hro2W5-V)l^l3V~8Eg}2@(WJGvSb;7UyAvs!|4!M?_)5NDNL@#5`7|s}c z>3PW}7^g9eWie%vAm_cv!Vpt119a5Ka&=N!O6;xO-1K_@M^z*lE^P%->~q<1xy6lm zwcuG8^YXdMQT?fsX7Xyh4|d65U8V@QXNb29mKOj1SD_BP2{Oar3$}fDUd?&PpKBX2 zE`m~HcyILL5W`?5jV-2|j77j{3iHM(1b>sX6u47$vCx!0!CFdC#UTLDpdBMQoQM&8 z(das-+`DzU%i{{ui5pd!qCpQN{;Ebj==X-n6Gx7IR4&VjD`j`_6GvA3ZXEa-sdd)t z!8}hU!S4{4NxF>4)iDFRX>?5L!-hU0t^+@%hIe8I__g8n47g)&>oXq#kuCbY_oh%} zxM-_c6d6ZK!fAutI>@}Sxh9W!kK%O;-n}QhJXLOc%E?Zgo88!Ml&w7at+SHC`V?wZ z7+W20q^gY>X>#J1j3i#D{Y)L<)~|>XK;W1ELOZm)>RWb{qkZ=o$<8uY&Bo``H%PB* zd^T=Lqx5kg5<}EqXIAOWOijr=JgY-FYGJ!rw(M%jmM2HuT*5tIsQD~6~Ullph>j4z`6Ud?(u@) z2os`GN3&10&Z{YN`T*blH8D&PtA$98ORGi-{~1eBv9Qnh07_Dqf%1j;cX7vmSCB^w zY<97vY~zF=GD3v@?7#z5hGMC$36)Q4OSCZphZdsbGGDE3nK)c5yGGEmiwaN4Jf_7Z z>BJ?XO|gGdrm%JsJMWhoV`}Q;#pM?vWDlKHWOocO9cb|sr)pyiSi;yXf%Sz&{JCXL zXbl1IY!KiaO-Q~En48}ULc*N1yw>5?n^jc`1yYg;d55Pql|i8OmmFC;ION5IP=5qY zHdHh3@AWa`3{Ef6SOzuu5F&eya@5L0Ta-zJezU zBy$vE)6aK%O4O6&l2`ntsaP3ypOk^M6_I5qQ_FhXga-Hmon6nSl=fWGvh%ZW<>s7b zu>Z0j%pb)~2H%8Nv`rwz6JZ*ie$yixMF z>_q~ObXQ*nr8MBPTZaLUazfIP-0S%54;Q{p4P7WCyq=-rIrDc9`>ce%bT%0MijB2=z%V|x!KYNp#b^zMBZ+8yyuPN%TR$dV2g)Slt4Rde8KSz3p!NIeP5hi~;7L;{zg>%)oieg?@5cY=xtd zC;ayfI2$Fkan=Q`hkaDDOmCWBpdg{K=ben7E*OxYRD-bxJ%hn$;-|+~IHX+_)zdpT zD5o_K^wWu&L-Ct)S3tJ(%JAzXZoKrQm2ZI(Y2R0~J;45BI7aPpHV4LT{Swt+?SJFT z<~Ozk>T&MS%E@gw2E;SmRP&B$?FgAucku3ZPhon+rh;1oT*n2C2*bS>onGgvKB@R~ zc2aw^;D2wlFr08JP{SOh7wh61ym#`KGBUd(a1Jhq?r2 zY$U`88eBO5RQ*QJ8l_+eK^L>&F|%-bcGRZG*^Sq-A>>d*;7$mv_|F^C-ua-MRGE+o zDhAg*{MfV+(sf{a=W9xryz)BL@HW;=s9A(%eCNq))9)@U#zV++>GKHz+h`@4d+Pta zkC_Fww=`eiZ{uMI?^dh|Cl;}_^F>i|*phBhnmDI1PLL2$386!Z()<% z5VDvh1}F6=%|AAF`L#jHMXM;6X(GzWBgJ;+!IvHfkA90CP|>AzU(FWd)f+!5KVyfH z_N=>8y691Hivan;>H6a`7xxXAoU^g0thfyPg2W0*Ygd=WtyPy+Y7)_J2i zCU(w)VYF}yPw6Lmab#R&Gl8m2DS`9mA-7HXffc%y56<{P2R==r@Vmg{Wv%5QxovXx zLw_18om>kOz6ius?hFVz`qHGr_-(^7L3@$-b@>=tU#sXYPx_)-Kl+jb(T;o3J+J2bdtDHi_Sa`a^fQ)l&YaA5mOB_7V#1Tr$rYU8l4w}bZ;KTlkWVr=!xA- zm2joaJYL*YG#<+JKQw>iNtOw^|g?Cl51AkhlR zuu&pke#e#$S$9?26GX@b%kgNsY!(}F)Giw32x6odOvVlsF^z({?wU!JZX}HOG7hfq zD_W+kd|jCSc=%uv(^Rtu4h<*h$O&HMr=w5FVeEz-BfX%Qghj z63A}bgp4`ESlUi#);I+GG^s$N&1K>86NajLW~@%pu@KIN@4Y&8D{cpiO>d%&>~VMEm}cr*>i4YLMVD^+}kJdlbEUg`&YmQ}V&jqDg}a z^0{1;Qe>Aw^yUJ*-jlGSENt~Q&lx5|C0FF>;E3JoMB8*g75zjU$;=ZZ*vgAQ+<^M3 z>v(wel!CU^#RK%!Fu=;$%_-6o?`5|k^ihRsTB-@1*Az*rfHgU0w(8dj8R8U#|NR)Y zd|dz&eNSlrB-DL6$NQ`)`2-zbf*T2E+~`QxS7N8k zlHS842X_7Nub%pe+qEi!SN9VS9*e;AB#-RMJ+WnO6M|;rEylCJMyEh*m!pw*{9!g6 z7EB?pOlW^d0c> zf-i}qB1h8X$6bBp)!jKEfyU&CQG-Cc`(%5JCmE;Nc@i~22MKwP^<7j7XF~g|AYeu1 z=hDQ9N3J*u^TUizzf0MKH1k}gQ1bBeuUP&I7IH<^x)&eZ#XYsxN+aex!LuyR=~w3P zaT_b<>a0wy!ddKL=6D%kSr#AA+FqrRQL&KfvWzJeKRxq%1&VEKGTO9|sDbK9M!XZ{ zxKt&%5`^~z2_dM`MpaZ&;qhHMam^7REooN?_l=)+`S+j&cQ4X`TZ>o$Mt(B5-n9_;3IWnP2LdtuXb;_u zv0in!R)#vPbP7icwgk{Ms)Qb;sl?@D^}wEAG2glTFl$?2?pIa8vQ%Lx2IX7K77bxN zg#IJxV_}_gf%v-j42xauJ9`WWqI9SX_aH@0YJ$iUPm{6`KW)zx{c~dXsN?Q^#B*3X zTp7=}`BzJAr@l}59@2=fRYifLc+eipy2Nt!WSoNlGu&COegUloyev)Vt^1+2Vgbkz z8_h^!osZ+$D0dvy+d2zcBV2|{2M8=7{0sRuo8Y%0kpOzSZXR1N50MmXyV@Rv0}e8C9KbE4neq+o zR-UnF;e*c;S(BG{S_x@4_Ii8@5okwYhvI0X^3-*$tU$h^IpU-?V5cPsdiA=g*m#@0buI#4mO08+IEfM%L0 z5!!E0Se!#?E_*kr&k%6Ec?ktMR?K$fYjy<~191N+*iy!>RdEC2=wM4z^6+6K%RqWd zkUH&PH@Tq=j~B6Afo-MKD%}F~Q9{tvaR}i-k(PVJ(>umGc zm|5u1#7?05ndnU`A9=6c1w_TJI#!OPtD-89IfiLk(-sB60XI%gTMD>%ram{GoAr+$ zpMcSQQs_$oj8*)EQW5?opewC_se6_AL%~AH!cj+6>CL{!6qFXtv()(yyTDHjt-tHR zPcskW9syTrvNeo#H?&qwlxT4Nq)VKYHM9b6HVGShCEw8*<7=6-78S1UoSuq zyAtGS4rudo(4){2XB&SH5+C{IhP!e|9fYAuKGx_&VGal3@5sVF(n~2~xq0}WMEq7V z#VG!nHd`U`-`i_X6onQ@+r?FMQsl7*3kQ?Fc2`kx8qI zSLBN`e%bq1Yynu1^02hJU;^ih;6#_InG!eKf4APhWi@+|ZYx#N8s^Gof9ZuOk0GYP zyK&IQn#=`|o#Zq_m-my@zKJoSntOe_F=xK}No(D`?LaIk0`brJNBr|Bk?jrRMG6Cw z=I9tyP8O-n?QDSTp>HmkKWHoJ_;B}hp)Q!vwNwW`jkeg8-X&YH@>$nJa%t-WZnf>w49FDCA;5qBe$yq^88ExPdKM~bJ(G^;IT24zS zsQe5%3SVl)RQIupjA)`os_VU!eam08%6`ybmaIBm7@5J_I!B z5KR*q*~QQx6Ui5Vti!050uIUWA@`n56bJKFS+1x@GYp=z9UI}t9&aJ6VR5?VUGu(} zg~<9CASQrCtq%DUE@1qcfD*V-O6f6{5k1lU)b-iIb{n_kmRT*D(#Ly6-dnrZxq(cx zUvDSJ*!NYq6X20Jx_~~o6JIqEIz$9J3aP`#J$ckOm3S!(vX`w zcI(yELfh4R%w9N)mcJqHM@icZlO-o@J)mLKx;6J@uobclBS%6$tTK$3V6 z8;ba|BsR~n{FD(vB@Gi8+_AB|#r8QPCzGy(wz*2onyL4fOOPb~+gUkKB4n%t_~$38 zn-PTJa%wp-y0i}8a%AX%=Yfl~$eppZ@D7y9~HI+kqQ)+2fZwN9uo$5Pve49mIjdb?u7cZ8H`%mB1J=PfPnLh*-k%py`H?3o zQZ4F~1Xug@4HPnZu`b9pexX>cm;^b_B-38DiW8X+eTc_EB|M%t@~XU9klbEe0O@h^yQOK*Tu+&*#C~^}e8ed?_Bp>@ z+tKVOP7Oku`Be$V zqjkBUOsxF0!cRmoJI;aa?Gg(VQv%D3RgF(Eeywm$sywL3hjJ<&o#Lyo#pZ^wL9P7U zab@PN@+KT5)_T>5oTy9~kY_OUhFwg_lDv?QVXOMumUh9?xYHhJLr*CxfCwT<_29A5 zfksY<+?6~jFwSPKEjj(>Nsxew?v^piMf)nVE}NRu*DO};LvyFg5+WCBBS`Z4+-k2) z+cag+i71wG?Z551)b$0NBDInFC~SFuQd{##o~!ga;vKuw;t{Y>dtK<(fp80JM-q@1 zsY@0(M`7fDDp|5?aOVuDnC+~`?Y}d$&T^O>Zf$Zltk8O1*VuTUp6 zQv16xdu+`ky0}}aLCM+v;SE!}>puA;j%BDaW4u|pqTkNwOg1zOIaRbnh=ykBaiNH+ z94!?{m)}_ze_pBy!B6{GlK!mHloEo{fg2VoCgq>A!{GK2-gRsI>tN`Km`H!=n(~0! zbZ^in1C>Kf?y8mIfw`0=%lZXt1Qt6NxHUrNC=~WS@H=~ak}{y9EBG8P6ZeGG+i7I#-%{eCI&|43HTLujegoH2NhRi{@xlk|;oxc^(@2qww zkk!vdjbE-V4t2EW#?*Hmzl;l9z}AxIZlIIIPwO-^+y$ooA+QN3Lhc~9n5u&xlv$uB z*S$r%Bxs-3Kd%sZV<-?(KT8+STYr#xl~w`=aj)7J`Cq)9Z$xIh-s&Tk5;t+LK9^8b zU|e)(uNTA8v!TMyPD&=GeyVblg85O7;zE_IinChB$EHa*7UdrKt{bvg0fEDSb#)?n zcJAASTG!Uy)X^(7v3#^3Os@cWjMAQJVaSPK9F#z1W${i>E>emCwr{`x!JWA?A8xK%wW2Dc zVns$iS!+kc-uq`;wW*k0yWe{Yf+jx2JLEpnA?Dd1*i_SYu`df8=sL{5qKT-_0*SVk?f@6I z|Ji=4bs=pws#k}0cVmAd+2}dDN8n55-EnFxm~D0iAK$8LRBF5nIR(L#_Vp?$pW52t z=_wDH8PC=fOpRsE+>s+rk`e~qn8`DTJ}m1Cwry4PV-hTNE+EPHxWw)U=5M6;RsRmm ze3$TEFz3C*MXs>CSuOcO>T6yGAKGY8pN{ssE=YFhwgFv6F?Uhn@F6k=3@1NnMdF1k zNgYhv94jKI>?u?q*Feps0aNJjX?qAX%^>Cu4#jAx=k55~%BCHZX02;AB&KKH6ZZ^j zZ9>31%WT5RKmI~;JFY-1Ww+=y>F$49yJp^;Q#@265j9^S`Be8oqr0%yyo-!>xT&^O zKb4>=VowYu%JJ4ed z%g&w9bu17`CcbIF@D(fy=`I)-_9s@VN6pb9R_bdofJZ3kT?&LSp+oZcA8vbVyW;;0 zQ(>h4_b`=zAzc5BUnFN}spRy-X_qBnpr`)_;o{)vM8M9$_8&1S1WZgnko12Y{|R{c zuN?0GmfCewOUD+Q4b6MGW=9JdW%uUY2oXg*tDS;%6{~L6^!!aAo~*gK-dNmW{i0^u zk-gB(SfUwA&eb2(5H?iE0q(l1^I03NFcC@eq$s!sdo|b`gJ2Rjz3Okbl=eyIM93x+d-&r(tv_pFfAxI%Z=tjZfIEbP0(LAk+ zUiFq1az6_TBRTjM(fWoL*}bHReyk3x*3mc`Rn_WYMpnI1v=7?q)->=`-IfPRVI@vI zF-T?@HtH{s%#C0ny)`$RD0Jv)o_|ldLYqgoj0opRz`xFmnW2ogFLeOvL@3}? z2poV}!6fE?^wrQJ7A1iLgVh`NdeN|!$4xb%i~ayb1NSdMHDIfUMUyn|lZT@LTlDY! za*kv$_=xLbF~QVVTr%Xo%(I2=@T17j4W?D-|EMzA9z;S<{ksL1!-ua-mRoJ5(3K`@ zHX-2bb#I_?wH;N`16Q8f@aoRPM^o`o>7nDThnr=&QQ9EzSdB+BWTDKs6Fkg`$)G_E zGKRX-`OufQ2z~p}RvIJnxOrhrTCTP?ws$lDq%+b`CR|vumuU<3vW$;pWc+|2s;D@N zRh29-6q+COm2Xax7;tz>&9q!Riv+!AKwig3j)zQzM>hr{wY()!VVBzwwPYPSw|AmP zzzLR4QzME+jqm1wY2+hLAaM4T4?@PI)*jHhYvS~PboF9DUhGT0B`IQCQ}irsY65j? zK-|nnyt=n&2M;HXLS&k-HmAu9GcsAjqltAU~x%z0OgF7LWn2r-bSQNskHa#>5@x3uY z6+=L(9t$e4FTGnN)6*or9YZ%Y{lXx=Y1Z##hzG zj>zJy7b|D=FLnjoE>4t1iO&(mtg7=pb2kO}a^Rh^k_pf(MPpYVEA$9u?Jum5>yU}`iN!oJ5 z=lcA`Dq)Qty9gT3@Pk$67-Cu>;6jXna@Mcx;0DgmJv?~7gt|}g!8Qt9wS`2%Y|1xG)gwG7tMmwICMDk1TjGPW}o^S6!MKvD%qW4=uigL4qbcc1)+ z6u6-3GO%}QH*nqE{O=5S-&>w|L<_?(0MgaveQcUZ*>%H$1fXsm*ZI=NebJg!S)VKF*Q*Y7t#N{XB|iU-r0