Merge pull request #114 from xssnick/v1-7-1

Better cell hashing & nft/jetton payload fixes & StoreOverlayNodes

README.md

@@ -3,7 +3,7 @@
 <img align="right" width="425px" src="https://github.com/xssnick/props/blob/master/logoimg.png?raw=true">
 
 [![Based on TON][ton-svg]][ton]
-![Coverage](https://img.shields.io/badge/Coverage-71.9%25-brightgreen)
+![Coverage](https://img.shields.io/badge/Coverage-72.4%25-brightgreen)
 
 Golang library for interacting with TON blockchain.
 

adnl/adnl.go

@@ -13,6 +13,7 @@ import (
  "github.com/xssnick/tonutils-go/tl"
  "log"
  "reflect"
+ "strings"
  "sync"
  "time"
 )
@@ -154,7 +155,7 @@ func (a *ADNL) processPacket(packet *PacketContent, ch *Channel) (err error) {
  }
 
  if seqno > a.confirmSeqno {
- a.confirmSeqno = uint64(*packet.Seqno)
+ a.confirmSeqno = seqno
  }
 
  if packet.ReinitDate != nil && *packet.ReinitDate > a.dstReinit {
@@ -536,8 +537,11 @@ func (a *ADNL) send(ctx context.Context, buf []byte) error {
 
  n, err := a.writer.Write(buf, dl)
  if err != nil {
- // it should trigger disconnect handler in read routine
- a.writer.Close()
+ // not close on io timeout because it can be triggered by network overload
+ if !strings.Contains(err.Error(), "i/o timeout") {
+ // it should trigger disconnect handler in read routine
+ a.writer.Close()
+ }
  return err
  }
  if n != len(buf) {

adnl/client.go

@@ -69,7 +69,7 @@ func Connect(ctx context.Context, addr string, peerKey ed25519.PublicKey, ourKey
  }
 
  return nil
- })
+ }, a.Close)
 
  go func() {
  if err = listenPacketsAsClient(a, conn); err != nil {

adnl/conn.go

@@ -14,10 +14,11 @@ type clientConn struct {
  mx sync.Mutex
 }
 
-func newWriter(writer func(p []byte, deadline time.Time) (err error)) *clientConn {
+func newWriter(writer func(p []byte, deadline time.Time) (err error), close func()) *clientConn {
  return &clientConn{
- closer: make(chan bool, 1),
- writer: writer,
+ onClose: close,
+ closer: make(chan bool, 1),
+ writer: writer,
  }
 }
 
@@ -41,6 +42,9 @@ func (c *clientConn) Close() error {
  if !c.closed {
  c.closed = true
  close(c.closer)
+ if h := c.onClose; h != nil {
+ go h() // to not lock
+ }
  }
 
  return nil

adnl/dht/client.go

@@ -325,11 +325,33 @@ func (c *Client) StoreAddress(ctx context.Context, addresses address.List, ttl t
  if err != nil {
  return 0, nil, err
  }
- return c.Store(ctx, []byte("address"), 0, data, ttl, ownerKey, copies)
-}
 
-func (c *Client) Store(ctx context.Context, name []byte, index int32, value []byte, ttl time.Duration, ownerKey ed25519.PrivateKey, atLeastCopies int) (copiesMade int, idKey []byte, err error) {
  id := adnl.PublicKeyED25519{Key: ownerKey.Public().(ed25519.PublicKey)}
+ return c.Store(ctx, id, []byte("address"), 0, data, UpdateRuleSignature{}, ttl, ownerKey, copies)
+}
+
+func (c *Client) StoreOverlayNodes(ctx context.Context, overlayKey []byte, nodes *overlay.NodesList, ttl time.Duration, copies int) (int, []byte, error) {
+ if len(nodes.List) == 0 {
+ return 0, nil, fmt.Errorf("0 nodes in list")
+ }
+
+ for _, node := range nodes.List {
+ err := node.CheckSignature()
+ if err != nil {
+ return 0, nil, fmt.Errorf("untrusted overlay node in list: %w", err)
+ }
+ }
+
+ data, err := tl.Serialize(nodes, true)
+ if err != nil {
+ return 0, nil, err
+ }
+
+ id := adnl.PublicKeyOverlay{Key: overlayKey}
+ return c.Store(ctx, id, []byte("nodes"), 0, data, UpdateRuleOverlayNodes{}, ttl, nil, copies)
+}
+
+func (c *Client) Store(ctx context.Context, id any, name []byte, index int32, value []byte, rule any, ttl time.Duration, ownerKey ed25519.PrivateKey, atLeastCopies int) (copiesMade int, idKey []byte, err error) {
  idKey, err = adnl.ToKeyID(id)
  if err != nil {
  return 0, nil, err
@@ -343,19 +365,22 @@ func (c *Client) Store(ctx context.Context, name []byte, index int32, value []by
  Index: index,
  },
  ID: id,
- UpdateRule: UpdateRuleSignature{},
+ UpdateRule: rule,
  },
  Data: value,
  TTL: int32(time.Now().Add(ttl).Unix()),
  }
 
- val.KeyDescription.Signature, err = signTL(val.KeyDescription, ownerKey)
- if err != nil {
- return 0, nil, fmt.Errorf("failed to sign key description: %w", err)
- }
- val.Signature, err = signTL(val, ownerKey)
- if err != nil {
- return 0, nil, fmt.Errorf("failed to sign value: %w", err)
+ switch rule.(type) {
+ case UpdateRuleSignature:
+ val.KeyDescription.Signature, err = signTL(val.KeyDescription, ownerKey)
+ if err != nil {
+ return 0, nil, fmt.Errorf("failed to sign key description: %w", err)
+ }
+ val.Signature, err = signTL(val, ownerKey)
+ if err != nil {
+ return 0, nil, fmt.Errorf("failed to sign value: %w", err)
+ }
  }
 
  kid, err := adnl.ToKeyID(val.KeyDescription.Key)

adnl/dht/client_test.go

@@ -11,8 +11,10 @@ import (
  "fmt"
  "github.com/xssnick/tonutils-go/adnl"
  "github.com/xssnick/tonutils-go/adnl/address"
+ "github.com/xssnick/tonutils-go/adnl/overlay"
  "github.com/xssnick/tonutils-go/liteclient"
  "github.com/xssnick/tonutils-go/tl"
+ "math/rand"
  "net"
  "reflect"
  "strconv"
@@ -47,7 +49,6 @@ func (m MockADNL) SetQueryHandler(handler func(msg *adnl.MessageQuery) error) {
 
 func (m MockADNL) SendCustomMessage(ctx context.Context, req tl.Serializable) error {
  return nil
-
 }
 
 func (m MockADNL) Answer(ctx context.Context, queryID []byte, result tl.Serializable) error {
@@ -585,7 +586,7 @@ func TestClient_Close(t *testing.T) {
  })
 }
 
-func TestClient_Store(t *testing.T) {
+func TestClient_StoreAddress(t *testing.T) {
  for i := 0; i < 15; i++ {
 
  addrList := address.List{
@@ -608,13 +609,6 @@ func TestClient_Store(t *testing.T) {
  Priority: 0,
  ExpireAt: 0,
  }
- tlAddrList, err := tl.Serialize(addrList, true)
- if err != nil {
- t.Fatal()
- }
-
- nameAddr := []byte("address")
- var index int32 = 0
 
  cliePrivK, err := hex.DecodeString("83590f541d37b783aa504049bab792696d12bbec3d23a954353300f816ca8b9693037f2613f6063869544caacac3eabbd7456e4d6e731478fccc961c137d1284")
  if err != nil {
@@ -707,7 +701,7 @@ func TestClient_Store(t *testing.T) {
  t.Fatal("failed to prepare test client, err: ", err)
  }
 
- count, _, err := cli.Store(context.Background(), nameAddr, index, tlAddrList, time.Hour, cliePrivK, 1)
+ count, _, err := cli.StoreAddress(context.Background(), addrList, time.Hour, cliePrivK, 1)
  if err != nil {
  t.Errorf(err.Error())
  }
@@ -717,3 +711,126 @@ func TestClient_Store(t *testing.T) {
  })
  }
 }
+
+func TestClient_StoreOverlayNodesIntegration(t *testing.T) {
+ pub, priv, err := ed25519.GenerateKey(nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ gateway := adnl.NewGateway(priv)
+ err = gateway.StartClient()
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ ctx, cancel := context.WithTimeout(context.Background(), 40*time.Second)
+ defer cancel()
+
+ dhtClient, err := NewClientFromConfigUrl(ctx, gateway, "https://ton-blockchain.github.io/global.config.json")
+ if err != nil {
+ t.Fatalf("failed to init DHT client: %s", err.Error())
+ }
+
+ time.Sleep(2 * time.Second)
+
+ id := make([]byte, 32)
+ rand.Read(id)
+
+ node, err := overlay.NewNode(id, priv)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ _, _, err = dhtClient.StoreOverlayNodes(ctx, id, &overlay.NodesList{
+ List: []overlay.Node{*node},
+ }, 5*time.Minute, 1)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ list, _, err := dhtClient.FindOverlayNodes(ctx, id)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ println("NUM", len(list.List))
+
+ if len(list.List) > 1 {
+ t.Fatal("list len")
+ }
+
+ if !bytes.Equal(list.List[0].ID.(adnl.PublicKeyED25519).Key, pub) {
+ t.Fatal("key not eq")
+ }
+}
+
+func TestClient_StoreAddressIntegration(t *testing.T) {
+ _, priv, err := ed25519.GenerateKey(nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ gateway := adnl.NewGateway(priv)
+ err = gateway.StartClient()
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ ctx, cancel := context.WithTimeout(context.Background(), 40*time.Second)
+ defer cancel()
+
+ dhtClient, err := NewClientFromConfigUrl(ctx, gateway, "https://ton-blockchain.github.io/global.config.json")
+ if err != nil {
+ t.Fatalf("failed to init DHT client: %s", err.Error())
+ }
+
+ time.Sleep(2 * time.Second)
+
+ id := make([]byte, 32)
+ rand.Read(id)
+
+ pub, key, _ := ed25519.GenerateKey(nil)
+
+ addrList := address.List{
+ Addresses: []*address.UDP{
+ {
+ net.IPv4(1, 1, 1, 1).To4(),
+ 11111,
+ },
+ {
+ net.IPv4(2, 2, 2, 2).To4(),
+ 22222,
+ },
+ {
+ net.IPv4(3, 3, 3, 3).To4(),
+ 333333,
+ },
+ },
+ Version: 0,
+ ReinitDate: 0,
+ Priority: 0,
+ ExpireAt: 0,
+ }
+
+ _, _, err = dhtClient.StoreAddress(ctx, addrList, 5*time.Minute, key, 2)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ kid, err := adnl.ToKeyID(adnl.PublicKeyED25519{
+ Key: pub,
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ res, _, err := dhtClient.FindAddresses(ctx, kid)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if len(res.Addresses) != 3 {
+ t.Fatal("addr len not 3")
+ }
+}

adnl/gateway.go

@@ -4,7 +4,6 @@ import (
  "bytes"
  "context"
  "crypto/ed25519"
- "encoding/hex"
  "fmt"
  "github.com/xssnick/tonutils-go/adnl/address"
  "github.com/xssnick/tonutils-go/tl"
@@ -268,7 +267,6 @@ func (g *Gateway) listen(rootId []byte) {
  g.mx.RUnlock()
 
  if proc == nil {
- Logger("unknown destination:", hex.EncodeToString(id))
  continue
  }
 
@@ -349,7 +347,7 @@ func (g *Gateway) registerClient(addr net.Addr, key ed25519.PublicKey, id string
  a.writer = newWriter(func(p []byte, deadline time.Time) (err error) {
  currentAddr := *(*net.Addr)(atomic.LoadPointer(&peer.addr))
  return g.write(deadline, currentAddr, p)
- })
+ }, a.Close)
  peer.client = a
 
  g.peers[id] = peer

adnl/overlay/manager-adnl.go

@@ -37,6 +37,7 @@ type ADNLWrapper struct {
  rootQueryHandler func(msg *adnl.MessageQuery) error
  rootDisconnectHandler func(addr string, key ed25519.PublicKey)
  rootCustomHandler func(msg *adnl.MessageCustom) error
+ unknownOverlayHandler func(msg *adnl.MessageQuery) error
 
  ADNL
 }
@@ -65,14 +66,21 @@ func (a *ADNLWrapper) SetCustomMessageHandler(handler func(msg *adnl.MessageCust
  a.rootCustomHandler = handler
 }
 
+func (a *ADNLWrapper) SetOnUnknownOverlayQuery(handler func(query *adnl.MessageQuery) error) {
+ a.unknownOverlayHandler = handler
+}
+
 func (a *ADNLWrapper) queryHandler(msg *adnl.MessageQuery) error {
- obj, over := unwrapQuery(msg.Data)
+ obj, over := UnwrapQuery(msg.Data)
  if over != nil {
  id := hex.EncodeToString(over)
  a.mx.RLock()
  o := a.overlays[id]
  a.mx.RUnlock()
  if o == nil {
+ if h := a.unknownOverlayHandler; h != nil {
+ return h(msg)
+ }
  return fmt.Errorf("got query for unregistered overlay with id: %s", id)
  }
 
@@ -113,7 +121,7 @@ func (a *ADNLWrapper) disconnectHandler(addr string, key ed25519.PublicKey) {
 }
 
 func (a *ADNLWrapper) customHandler(msg *adnl.MessageCustom) error {
- obj, over := unwrapMessage(msg.Data)
+ obj, over := UnwrapMessage(msg.Data)
  if over != nil {
  id := hex.EncodeToString(over)
  a.mx.RLock()