Merge pull request #268 from KitHat/master

 ST-600 & ST-601 -- remove utxo from state and limit the number of utxo's returned

sovtoken/sovtoken/constants.py

@@ -24,3 +24,6 @@
                           TokenTransactions.XFER_PUBLIC.value}
 ACCEPTABLE_QUERY_TYPES = {TokenTransactions.GET_UTXO.value, }
 ACCEPTABLE_ACTION_TYPES = {}
+
+UTXO_LIMIT = 1000
+NEXT_SEQNO = "next"

sovtoken/sovtoken/request_handlers/read_req_handler/get_utxo_handler.py

@@ -1,5 +1,11 @@
+try:
+    import ujson as json
+except ImportError:
+    import json
+from copy import deepcopy
+
 from sovtoken import TokenTransactions
-from sovtoken.constants import ADDRESS, OUTPUTS, TOKEN_LEDGER_ID
+from sovtoken.constants import ADDRESS, OUTPUTS, TOKEN_LEDGER_ID, NEXT_SEQNO, UTXO_LIMIT
 from sovtoken.messages.txn_validator import txt_get_utxo_validate
 from sovtoken.request_handlers.token_utils import parse_state_key
 from sovtoken.types import Output
@@ -13,6 +19,7 @@
 from plenum.server.database_manager import DatabaseManager
 from plenum.server.request_handlers.handler_interfaces.read_request_handler import ReadRequestHandler
 from state.trie.pruning_trie import rlp_decode
+from stp_core.config import MSG_LEN_LIMIT
 
 
 class GetUtxoHandler(ReadRequestHandler):
@@ -56,10 +63,21 @@ def get_result(self, request: Request):
                 continue
             outputs.add(Output(addr, int(seq_no), int(amount)))
 
+        utxos = outputs.sorted_list
+        next_seqno = None
+        if len(utxos) > UTXO_LIMIT:
+            next_seqno = utxos[UTXO_LIMIT].seqNo
+            utxos = utxos[:UTXO_LIMIT]
+
         result = {f.IDENTIFIER.nm: request.identifier,
-                  f.REQ_ID.nm: request.reqId, OUTPUTS: outputs.sorted_list}
-        if proof:
-            result[STATE_PROOF] = proof
+                  f.REQ_ID.nm: request.reqId, OUTPUTS: utxos}
 
         result.update(request.operation)
+        if next_seqno:
+            result[NEXT_SEQNO] = next_seqno
+        if proof:
+            res_sub = deepcopy(result)
+            res_sub[STATE_PROOF] = proof
+            if len(json.dumps(res_sub)) <= MSG_LEN_LIMIT:
+                result = res_sub
         return result

sovtoken/sovtoken/request_handlers/token_utils.py

@@ -13,7 +13,7 @@
 
 def spend_input(state, utxo_cache: UTXOCache, address, seq_no, is_committed=False):
     state_key = create_state_key(address, seq_no)
-    state.set(state_key, b'')
+    state.remove(state_key)
     utxo_cache.spend_output(Output(address, seq_no, None),
                             is_committed=is_committed)
 

sovtoken/sovtoken/test/helper.py

@@ -13,6 +13,10 @@
 from sovtoken.test.wallet import TokenWallet
 
 
+def libsovtoken_address_to_address(addr):
+    return addr[8:]
+
+
 def xfer_request(inputs, outputs, extra_data=None):
     payload = {
         TXN_TYPE: XFER_PUBLIC,

sovtoken/sovtoken/test/req_handlers/get_utxo_req_handler/conftest.py

@@ -4,6 +4,8 @@
 
 from sovtoken import TOKEN_LEDGER_ID, TokenTransactions
 from sovtoken.request_handlers.read_req_handler.get_utxo_handler import GetUtxoHandler
+from sovtoken.request_handlers.token_utils import create_state_key
+from sovtoken.test.helper import libsovtoken_address_to_address
 
 from plenum.test.helper import sdk_json_to_request_object
 
@@ -22,3 +24,10 @@ def get_utxo_request(looper, payment_address, wallet):
     get_utxo_request = json.loads(get_utxo_request)
     get_utxo_request = sdk_json_to_request_object(get_utxo_request)
     return get_utxo_request
+
+
+@pytest.fixture(scope="module")
+def insert_over_thousand_utxos(db_manager, payment_address):
+    token_state = db_manager.get_state(TOKEN_LEDGER_ID)
+    for i in range(1200):
+        token_state.set(create_state_key(libsovtoken_address_to_address(payment_address), i), str(i).encode())

sovtoken/sovtoken/test/req_handlers/get_utxo_req_handler/test_get_utxo_get_result.py

@@ -1,27 +1,28 @@
 import pytest
 from sovtoken.constants import OUTPUTS, ADDRESS
 from sovtoken.request_handlers.token_utils import create_state_key
+from sovtoken.test.helper import libsovtoken_address_to_address
 
 from plenum.common.constants import STATE_PROOF
 
 
 @pytest.fixture(scope="module", autouse=True)
 def add_utxo(payment_address, get_utxo_handler):
-    get_utxo_handler.state.set(create_state_key(payment_address.replace("pay:sov:", ""), 1), "3".encode())
+    get_utxo_handler.state.set(create_state_key(libsovtoken_address_to_address(payment_address), 1), "3".encode())
 
 
 def test_get_utxo_request_has_utxos(get_utxo_request, get_utxo_handler, payment_address, add_utxo):
     result = get_utxo_handler.get_result(get_utxo_request)
     assert result[STATE_PROOF]
     assert result[OUTPUTS]
     assert len(result[OUTPUTS]) == 1
-    assert result[OUTPUTS][0].address == payment_address.replace("pay:sov:", "")
+    assert result[OUTPUTS][0].address == libsovtoken_address_to_address(payment_address)
     assert result[OUTPUTS][0].amount == 3
     assert result[OUTPUTS][0].seqNo == 1
 
 
 def test_get_utxo_request_no_utxos(get_utxo_request, get_utxo_handler, payment_address_2, add_utxo):
-    get_utxo_request.operation[ADDRESS] = payment_address_2.replace("pay:sov:", "")
+    get_utxo_request.operation[ADDRESS] = libsovtoken_address_to_address(payment_address_2)
     result = get_utxo_handler.get_result(get_utxo_request)
     assert result[STATE_PROOF]
     assert len(result[OUTPUTS]) == 0

sovtoken/sovtoken/test/req_handlers/get_utxo_req_handler/test_get_utxo_get_result_big_number_of_utxos.py

@@ -0,0 +1,15 @@
+from sovtoken.constants import OUTPUTS
+from sovtoken.test.helper import libsovtoken_address_to_address
+
+from plenum.common.constants import STATE_PROOF
+
+
+def test_get_utxo_request_has_utxos(get_utxo_request, get_utxo_handler, payment_address, insert_over_thousand_utxos):
+    result = get_utxo_handler.get_result(get_utxo_request)
+    assert result[STATE_PROOF]
+    assert result[OUTPUTS]
+    assert len(result[OUTPUTS]) == 1000
+    for i in range(1000):
+        assert result[OUTPUTS][i].address == libsovtoken_address_to_address(payment_address)
+        assert result[OUTPUTS][i].seqNo == i
+        assert result[OUTPUTS][i].amount == i

sovtoken/sovtoken/test/req_handlers/mint_req_handler/test_mint_req_handler_update_state.py

@@ -1,11 +1,13 @@
+from sovtoken.test.helper import libsovtoken_address_to_address
+
 
 def test_mint_handler_update_state_valid_txn(mint_txn, mint_handler, payment_address):
     mint_handler.update_state(mint_txn, None, None, is_committed=True)
 
     token_state = mint_handler.state
     utxo_cache = mint_handler.database_manager.get_store("utxo_cache")
 
-    assert int(token_state.get((payment_address[8:] + ":1").encode(), isCommitted=False)) == 10
-    assert utxo_cache.get(payment_address[8:].encode()).decode() == '1:10'
+    assert int(token_state.get((libsovtoken_address_to_address(payment_address) + ":1").encode(), isCommitted=False)) == 10
+    assert utxo_cache.get(libsovtoken_address_to_address(payment_address).encode()).decode() == '1:10'
 
 

sovtoken/sovtoken/test/req_handlers/xfer_req_handler/conftest.py

@@ -6,6 +6,8 @@
 from sovtoken.sovtoken_auth_map import sovtoken_auth_map
 from base58 import b58encode_check
 from indy.payment import build_payment_req
+from sovtoken.test.helper import libsovtoken_address_to_address
+
 from indy_common.test.auth.conftest import write_auth_req_validator, constraint_serializer, config_state, idr_cache
 from plenum.common.txn_util import append_txn_metadata
 from plenum.test.helper import sdk_json_to_request_object
@@ -20,7 +22,7 @@ def xfer_handler(utxo_cache, db_manager, write_auth_req_validator, mint_tokens):
 
 @pytest.fixture(scope="module")
 def mint_tokens(payment_address, utxo_cache, db_manager):
-    addr = payment_address[8:]
+    addr = libsovtoken_address_to_address(payment_address)
     utxo_cache.set(addr, "1:10".encode())
     db_manager.get_state(TOKEN_LEDGER_ID).set((addr + ":1").encode(), "10".encode())
 

sovtoken/sovtoken/test/req_handlers/xfer_req_handler/test_xfer_handler_dynamic_validation.py

@@ -1,5 +1,6 @@
 import pytest
 from sovtoken.exceptions import InsufficientFundsError, ExtraFundsError, InvalidFundsError
+from sovtoken.test.helper import libsovtoken_address_to_address
 
 
 def test_xfer_handler_dynamic_validation_valid(xfer_handler, xfer_request):
@@ -22,5 +23,5 @@ def test_xfer_handler_dynamic_validation_utxo_not_exists(xfer_handler, invalid_a
                                                          payment_address):
     with pytest.raises(InvalidFundsError,
                        message="InvalidFundsError(\"seq_nos {{2}} are not found in list of seq_nos_amounts for "
-                               "address {} -- current list: ['1', '10']\",)".format(payment_address[8:])):
+                               "address {} -- current list: ['1', '10']\",)".format(libsovtoken_address_to_address(payment_address))):
         xfer_handler.dynamic_validation(invalid_amount_xfer_request_utxo_does_not_exist)

sovtoken/sovtoken/test/req_handlers/xfer_req_handler/test_xfer_handler_update_state.py

@@ -1,8 +1,11 @@
+from sovtoken.test.helper import libsovtoken_address_to_address
+
+
 def test_xfer_handler_update_state(xfer_handler, xfer_txn, payment_address_2):
     xfer_handler.update_state(xfer_txn, None, None)
 
     token_state = xfer_handler.state
     utxo_cache = xfer_handler.utxo_cache
 
-    assert int(token_state.get((payment_address_2[8:] + ":2").encode(), isCommitted=False)) == 10
-    assert utxo_cache.get(payment_address_2[8:]) == '2:10'
+    assert int(token_state.get((libsovtoken_address_to_address(payment_address_2) + ":2").encode(), isCommitted=False)) == 10
+    assert utxo_cache.get(libsovtoken_address_to_address(payment_address_2)) == '2:10'

sovtoken/sovtoken/test/test_get_utxo.py

@@ -3,12 +3,14 @@
 import pytest
 
 from base58 import b58encode_check
+from sovtoken.request_handlers.token_utils import create_state_key
+from sovtoken.test.helper import libsovtoken_address_to_address
 from sovtoken.test.helpers.helper_general import utxo_from_addr_and_seq_no
 
 from plenum.common.exceptions import RequestNackedException
 from plenum.common.txn_util import get_seq_no, get_payload_data
 from plenum.common.util import randomString
-from sovtoken.constants import OUTPUTS, ADDRESS, AMOUNT, PAYMENT_ADDRESS
+from sovtoken.constants import OUTPUTS, ADDRESS, AMOUNT, PAYMENT_ADDRESS, TOKEN_LEDGER_ID, NEXT_SEQNO, SEQNO
 
 
 @pytest.fixture
@@ -116,3 +118,30 @@ def test_get_utxo_utxos_in_order(helpers, addresses):
             seq_nos.append(output["seqNo"])
 
         assert seq_nos == sorted(seq_nos)
+
+
+def test_get_more_then_thousand_utxos(helpers, addresses, nodeSetWithIntegratedTokenPlugin):
+    """
+    test if we send more have more than a 1000 UTXO's we still receive a response.
+    """
+
+    _, address_2 = addresses
+
+    states = [n.db_manager.get_state(TOKEN_LEDGER_ID) for n in nodeSetWithIntegratedTokenPlugin]
+    utxos = []
+
+    for i in range(1200):
+        amount = randint(1, 5)
+        key = create_state_key(libsovtoken_address_to_address(address_2), i+5)
+        utxos.append((key, amount))
+        for state in states:
+            state.set(key, str(amount).encode())
+
+    request = helpers.request.get_utxo(address_2)
+    responses = helpers.sdk.send_and_check_request_objects([request])
+    for response in responses:
+        result = response[1]['result']
+        assert len(result[OUTPUTS]) == 1000
+        for output in result[OUTPUTS]:
+            assert (create_state_key(output[ADDRESS], output[SEQNO]), output[AMOUNT]) in utxos
+        assert result.get(NEXT_SEQNO, None)

sovtoken/sovtoken/test/test_public_xfer_1.py

@@ -1,10 +1,11 @@
 import pytest
+from sovtoken.request_handlers.token_utils import create_state_key
 
 from plenum.common.txn_util import get_seq_no
 from plenum.common.exceptions import RequestNackedException
 from plenum.common.types import OPERATION
-from sovtoken.constants import SIGS, ADDRESS, SEQNO, AMOUNT, OUTPUTS, PAYMENT_ADDRESS, INPUTS
-from sovtoken.test.helper import user1_token_wallet
+from sovtoken.constants import SIGS, ADDRESS, SEQNO, AMOUNT, OUTPUTS, PAYMENT_ADDRESS, TOKEN_LEDGER_ID, INPUTS
+from sovtoken.test.helper import user1_token_wallet, libsovtoken_address_to_address
 
 
 @pytest.fixture
@@ -33,6 +34,23 @@ def initial_mint_inner(helpers, addresses_inner):
     return result
 
 
+def test_state_after_xfer(helpers, initial_mint, addresses, nodeSetWithIntegratedTokenPlugin):
+
+    mint_seq_no = get_seq_no(initial_mint)
+    [address1, address2, *_] = addresses
+
+    inputs = helpers.general.get_utxo_addresses([address1])
+    inputs = [utxo for utxos in inputs for utxo in utxos]
+    outputs = [{"address": address2, "amount": 100}]
+
+    helpers.general.do_transfer(inputs, outputs)
+    key = create_state_key(libsovtoken_address_to_address(address1), mint_seq_no)
+
+    for n in nodeSetWithIntegratedTokenPlugin:
+        res = n.db_manager.get_state(TOKEN_LEDGER_ID).get(key)
+        assert not res
+
+
 def test_multiple_inputs_with_1_incorrect_input_sig(  # noqa
     helpers,
     addresses,

sovtoken/sovtoken/test/test_token_req_handler.py

@@ -4,6 +4,7 @@
 import base58
 import pytest
 from sovtoken.request_handlers.token_utils import commit_to_utxo_cache
+from sovtoken.test.helper import libsovtoken_address_to_address
 from sovtoken.test.helpers.helper_general import utxo_from_addr_and_seq_no
 
 from plenum.common.constants import (IDENTIFIER, STATE_PROOF,
@@ -196,8 +197,8 @@ def test_token_req_handler_apply_xfer_public_success(
     outputs = [{"address": address1, "amount": 30}, {"address": address2, "amount": 30}]
     request = helpers.request.transfer(inputs, outputs)
     # test xfer now
-    address1 = address1.replace("pay:sov:", "")
-    address2 = address2.replace("pay:sov:", "")
+    address1 = libsovtoken_address_to_address(address1)
+    address2 = libsovtoken_address_to_address(address2)
     utxo_cache = xfer_handler_a.database_manager.get_store(UTXO_CACHE_LABEL)
     pre_apply_outputs_addr_1 = utxo_cache.get_unspent_outputs(address1)
     pre_apply_outputs_addr_2 = utxo_cache.get_unspent_outputs(address2)
@@ -235,11 +236,11 @@ def test_token_req_handler_apply_MINT_PUBLIC_success(
     outputs = [{"address": address, "amount": 100}]
     request = helpers.request.mint(outputs)
     utxo_cache = mint_handler.database_manager.get_store(UTXO_CACHE_LABEL)
-    pre_apply_outputs = utxo_cache.get_unspent_outputs(address.replace("pay:sov:", ""))
+    pre_apply_outputs = utxo_cache.get_unspent_outputs(libsovtoken_address_to_address(address))
     assert pre_apply_outputs == []
     # Applies the MINT_PUBLIC transaction request to the UTXO cache
     mint_handler.apply_request(request, CONS_TIME, None)
-    post_apply_outputs = utxo_cache.get_unspent_outputs(address.replace("pay:sov:", ""))
+    post_apply_outputs = utxo_cache.get_unspent_outputs(libsovtoken_address_to_address(address))
     assert post_apply_outputs[0].amount == 100
 
 
@@ -300,9 +301,9 @@ def test_token_req_handler_update_state_XFER_PUBLIC_success(
     xfer_handler_a.dynamic_validation(request)
     xfer_handler_a.update_state(txn, None, request)
 
-    state_key = TokenReqHandler.create_state_key(address1.replace("pay:sov:", ""), seq_no)
+    state_key = TokenReqHandler.create_state_key(libsovtoken_address_to_address(address1), seq_no)
     utxo_cache = xfer_handler_a.database_manager.get_store(UTXO_CACHE_LABEL)
-    key = utxo_cache._create_key(Output(address1.replace("pay:sov:", ""), seq_no, 60))
+    key = utxo_cache._create_key(Output(libsovtoken_address_to_address(address1), seq_no, 60))
     assert utxo_cache._store._has_key(key)
     try:
         xfer_handler_a.state.get(state_key, False)
@@ -384,7 +385,7 @@ def test_token_req_handler_get_result_success(
     results = get_utxo_handler.get_result(request)
 
     state_proof = results.pop(STATE_PROOF)
-    address1 = address1.replace("pay:sov:", "")
+    address1 = libsovtoken_address_to_address(address1)
     assert state_proof
     assert results == {
         ADDRESS: address1,
@@ -422,12 +423,12 @@ def test_token_req_handler_get_all_utxo_success(
 
     assert state_proof
     assert results == {
-        ADDRESS: address1.replace("pay:sov:", ""),
+        ADDRESS: libsovtoken_address_to_address(address1),
         TXN_TYPE: GET_UTXO,
         OUTPUTS: [
-            Output(address=address1.replace("pay:sov:", ""), seq_no=1, value=40)
+            Output(address=libsovtoken_address_to_address(address1), seq_no=1, value=40)
         ],
-        IDENTIFIER: base58.b58encode(base58.b58decode_check(address1.replace("pay:sov:", ""))).decode(),
+        IDENTIFIER: base58.b58encode(base58.b58decode_check(libsovtoken_address_to_address(address1))).decode(),
         TXN_PAYLOAD_METADATA_REQ_ID: request.reqId
     }
 

sovtoken/sovtoken/token_req_handler.py

@@ -273,7 +273,7 @@ def sum_outputs(request: Request) -> int:
     @staticmethod
     def spend_input(state, utxo_cache, address, seq_no, is_committed=False):
         state_key = TokenReqHandler.create_state_key(address, seq_no)
-        state.set(state_key, b'')
+        state.remove(state_key.decode())
         utxo_cache.spend_output(Output(address, seq_no, None),
                                 is_committed=is_committed)
 

sovtoken/sovtoken/types.py

@@ -3,7 +3,7 @@
 
 
 class Output:
-    def __init__(self, address: str, seq_no: str, value: Optional[int]):
+    def __init__(self, address: str, seq_no: int, value: Optional[int]):
         self.address = address
         self.seqNo = seq_no
         self.amount = value

sovtokenfees/sovtokenfees/test/req_handlers/domain_fee_handler/test_domain_fee_handler_update_state.py

@@ -1,3 +1,5 @@
+from sovtoken.test.helper import libsovtoken_address_to_address
+
 from indy_common.constants import NYM
 
 
@@ -7,7 +9,8 @@ def test_domain_fee_handler_update_state(domain_fee_handler, nym_request_with_fe
     token_state = domain_fee_handler.token_state
     utxo_cache = domain_fee_handler.utxo_cache
 
-    assert int(token_state.get((payment_address[8:] + ":2").encode(), isCommitted=False)) == 9
-    assert utxo_cache.get(payment_address[8:]) == '2:9'
+    assert int(token_state.get((libsovtoken_address_to_address(payment_address) + ":2").encode(), isCommitted=False)) == 9
+    assert not token_state.get((libsovtoken_address_to_address(payment_address) + ":1").encode(), isCommitted=False)
+    assert utxo_cache.get(libsovtoken_address_to_address(payment_address)) == '2:9'
     assert domain_fee_handler._fees_tracker.fees_in_current_batch == 1
     assert domain_fee_handler._fees_tracker.has_deducted_fees(NYM, 1)