From 6bc333edda42be631ac4065ca7a42db9ac28c851 Mon Sep 17 00:00:00 2001 From: Joseph Wyman Date: Thu, 17 Oct 2024 05:21:23 -0400 Subject: [PATCH 01/85] add privacy notes --- NOTES.md | 690 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 690 insertions(+) diff --git a/NOTES.md b/NOTES.md index ed99c5fa..856e3380 100644 --- a/NOTES.md +++ b/NOTES.md @@ -178,3 +178,693 @@ The linux system that I build the app with does not have to be the one that I te - helpful walk through https://www.makeworld.space/2021/10/linux-wine-pyinstaller.html - after the exe is built you can build with electron-builder targeting the windows build. + + + +# Privacy + +## part one https://21ideas.org/en/privacy/oxt-1/ +- Most traciking techniques are reliant on heuristics and evaluating the flow of bitcoin. +- Bitcoin transaction activity is pseudonymous, not anonymous. A user's true name and personal identification information is not included in the Bitcoin protocol. +- transaction include transparent amounts to a transparent address. +- The most comon form of chain analysis is focused on identification of transaction change outputs. The process is based on a series of heuristics that can be used to follow a user's activity over multiple transctions. +- If onchain activity leads to an identified economic entity wallet cluster, investigators may be able to obtain user PII associtated with the observed transaction activity. +1. Change outputs can be used to track a users activity on the blockchain. +2. The intersection of this activity with entities that obtain P11 links observed blockchain activity with a possible real identity +- What are Heuristics? + - They are rules of thumb used to make decisions under uncertain conditions. +- Chain analysis is based on heuristics. +- Change detection is used to cluster separate address by the common input odwnership heuristic. +- What is the difference between an address and a utxo? + - UTXOs are pieces of bitcoin paid to an address. + - an address can receive multiple utxos, but a utxo can not have multiple addresses + +- Example transactions + - Simple spend + - make up about 50% of bitcoin transactions in recent blocks + - user make a payment and receives a change output. + - traits + - Number of Inputs: 1 or more + - Number of Outputs:2 + - common interpretation: 1 payment output, 1 change output + + - sweep + - Spend the entirety of a single utxo to a new address + - traits + - Number of inputs: 1 + - Number of Outputs: 1 + - common intepretation: Possible self transfer + + - Consolidation Spend + - Combine multiple UTXOs into a single utxo. These are rarely true payments, because a normal payment has a change output. + - traits + - Number of Inputs: > 1 + - Number of Outputs: 1 + - Common Interpretation: Possible self-transfer + + - Batch Spend + - Most likely performed by exchanges and include 1 or more inputs and many outputs. These transactions aim to save on miner fees by making as many payments as possible in a single transaction. + - Traits + - Number of Inputs: >1 + - Number of Outputs: MANY + - Common Interpretation: Large economic activity, likely exchange. + + - Multi-party Transactions (Coinjoin) + - Multi-party transxctions involve collaboration between many useres to perform a signle transaction that improves participant privacy. These transactions are easily identified by their equal output amounts + - Traits: + - Number of inputs: Many + - Number of outputs Many + - Output profile: Number of identical outputs is a proxy for number of participants + + + +Much of traditional chain analysis is based on detecting this change output. If a change output can be successfully detected, a signle user's activity can be tracked across a series of transctions + +Most common ways of detecting change, and building a single user from it +- Address Reuse + - multiple uses of the same address are signs of activity of the same private key. + - if one output is to a new address and the remaining output is to the same address as the input address, we know that the reused address is the change output. +- Round Number Payment Heuristic + - it is difficult for a user to generate a change output for a round number amount since it is input amount - payment amount - tx size * network fee rate + - in a simple spend the round number output is the likely payment which makes the remaining output the change output + - so more likely the .16 btc is the receiver and the .13432343 is the change. +- Same Script type heuristic. + - If one uotput is to the same type as the input and the remaining output is to the new address script type, the output to the new address script type is the likely payment, and the same the likely change +- Largest Output Amount Heuristic. + - The largest output amount is the likely chagne output. This is the weakest heuristic. + + +What are these heuristics used to do? +- They allow for tracking a single users activity over multiple transactions + + +## Part two https://21ideas.org/en/privacy/oxt-2/ +- external data can be used to improve confidence in change detection. +- examples of external data + - address reuse over multiple transactions + - co-spending from multiple addresses in the same transaction (wallet clustering) + - multi-sig scripts which are revelaed after a utxo is spent. + - wallet fingerprinting over a series of transctions. + - outputs spent to labelled clusters and ip addresses + - volume, timing and other transaction pattern recognition anaylse. + + +- multiple inputs to same transactions can be thought of as owned by the same person + - an analyst can assume that each address used as an input to a transaction, is controlled by the same private key or wallet software. often called the merge input heuristic. +- Even if a cluster is identified it is still anonymous until real world entity activity is attached to it. +- Exchanges recieving + - The most damaging to transaction privacy is when a simple spend is made to a wallet cluster that has been identified as a centralized service. + - When a simple spend is made to an exchange, all change detection ambiguiity is lost. The exchange is clearly the receiver, and the other output is the users. + - this isn't about identifying per say but about wallet clustering. + + +# Part three defense against chain analytics +- https://21ideas.org/en/privacy/oxt-3/ +- payments priced in fiat are not likely to have round inputs. therefore protect against that type of analysis. +- the same output position for change from a wallet is not goood, maked it easier to tell what hte chagne is + + +Breaking deterministic links +- breaking deterministic links and creating on chain ambiguity requires a specific transaction structure. + - determinism is dependent on the number of transaction inputs, outputs and the BTC amounts of each UTXO.- +- boltzmann algo created by LAurenMT uses the CoinKoin suduku concept to evaluate transaction for several privacy related metrics + - https://github.com/Samourai-Wallet/boltzmann/blob/master/boltzmann/linker/txos_linker.py + - https://medium.com/@laurentmt/introducing-boltzmann-85930984a159 + - https://www.youtube.com/watch?v=CYIDAqMSc4A&ab_channel=WasabiWallet + + +- Does a link between inputs and equal outputs exist in a coinjoin? + - yes but these links are porbabalistic not deterministic. +What does the boltzmann algo calculate? + - a link probability matrix, for the relationship between a transactions inputs and outputs. + - this will give you a percentage out of 100% that the output owner can be linked to the input owner + +- Botlman effectively uses subset analysis to ask the question: Are there multiple ways (interpretations) a transction's inputs could have paid its outputs. + +# Part Four privacy enhancing technologies +- https://21ideas.org/en/privacy/oxt-4/ +- some pre context is usually needed for performing an anaylsis, like having information from an exchanged share with an analyszer. +- there are two investigation directions, you can search for the source of funds by searching the past history of the target utxo + - Source evaluation for transaction chains with a single input are fairly simple to perform, because there is no “decision” to be made in evaluating which input UTXO path to follow. However, multi-input transactions present analysts with multiple sources to evaluate + - this type of anaylsis will end though once an exchange is hit, you usually don't go back past an exchange to find another source since an exchange will change the source user of the input and output. + - UTXO flows should not be tracked across custodial services. It is highly unlikely that a deposit UTXO will be used to payout to the entity making the deposit +- privacy implications of making payments + - when making a payment you will reveal some information to a counter party. you will revela more the more you have in a utxo above the payment of your item. + - In addition, payments made by a sender allow for a recipient to assess the senders past transaction history. Payments also allow a sender to evaluate a recipients future spending of their received payment. +- MUCH OF CHAIN ANAYLSIS IS BASED ON + - analysts needing a starting point + - the issues of the transaction graph + - change detection + - clustering separate addresses by the common input ownership heuristic + - common input ownership heuristic is if a transaction has more than one input then all those inputs are owned by the same entity + +- techniques for denyability + - Stealth Addresses — Denying a Start Point by using BIP 47. + - examples of starting points are like a user posting an address online for donations. and that beign linked to a person. from that address you can analyse a user’s activity including received payments, total address balance, and spending patterns. all moving forward. + - all shared addresses + - coin control. + - Combining inputs from multiple sources in future spends can allow payment senders to evaluate the transaction histories of additional UTXOs combined with their spent UTXO. + - how to practice coin control? + - Labelling received payment UTXOs. At a minimum labelling should include the sender and reason for payment. + - “Marking Do Not Spend”. To prevent a wallet from accidentally including a UTXO in a future payment, UTXOs can be made inactive for inclusion in future spends. + - Sending Individual (Selective UTXO Activation). UTXOs can be selectively spent. + - Richocet transactions + - aka dummy transactions. + - Ricochet is a simple tool that automatically adds “hops”, or dummy transactions between an origin UTXO and payment destination + - Ricochet transactions do not obfuscate source of funds or break the transaction graph. However, these transactions put distance between a payment destination and previous UTXO histories. +- Stonewall and Stonewall x2 — Payments Made Safe + - they are simulated coinjoins. either with one user or two users. + - I think at least two outputs will have the same utxo amount. + - stonewall x2 is two users putting utxos into a transactions and both receiving outputs utxos. +- Stoweaway + - when a payment includes a utxo from the recipient. This way the multiple inputs look like they are coming from the same person when infact they are not. + - so a two input two output transaction is read as a normal transaction but that is actually not true. + - the true transaction payment amount will also be hidden since the receiver is adding an amount as well. +- Coinjoins. + - payment recipients can coinjoin to break the link between their payment receipt and future spending. In otherwords, sending UTXOs received as payment through a coinjoin establishes forward privacy. + - coinjoins do not include “unmixed change” outputs within the coinjoin transaction that can be used to continue to track user activity. + +- steps for good privacy + - not linking blockchain activities to an online persona, + - avoiding address reuse, + - segregating UTXOs with different histories, + - establishing forward privacy with coinjoin, + - and using the advanced spending tools previously discussed to undermine chain analysis heursitics. + + + + +# related videos +- https://www.youtube.com/watch?v=06aMJuF1ygU&list=PLIBmWVGQhizLrPjpFMN5bQdbOZRxCQXUg&index=3&ab_channel=SamouraiWallet +- another reason why address reuse is bad is because anyone can look up my address and see my wallet balance. as well they could see every tx i hae made or received +- change is essentially money you have already spent once, you just spent it in another utxo, so don't go spending it in another one unless you want to link them together + - If a transaction has multiple intra-UTXO flow interpretations, Boltzmann will score the transaction’s entropy greater than or equal to 0. The concept of entropy originates from a thermodynamic mental model. In this model, the number of interpretations can be thought of as microstates of the overall transaction macrostate. + - entropy can be seen as a metric measuring the analysts lack of knowledge about the exact confirguration of the transaction being observed + +- coinjoin properties are evidence that a transcation has multiple users, contributing inputs to the transaction. Therefore transactions with entropy have coinjoin properties and broken deterministic links + +- a coinjoin is clearly observed on chain and therefore the person looking at it knows there are multiple users and can be sure they can not know for certian which input is related ot the output. This is like encryption, you know the message exists but an outside observer does not know the details of it. +- with stoweaway aka payjoins, they may not know it is a privacy based transaction and therefore incorrectly assume each of the inputs are controller by the same user. an outside observer is not aware that the "message" happened aka a secret message hiding technique + - payjoins often result in false cluster on chain footprint, since the observer does not know they happened + + +- summary + - Change detection heuristics can be defeated by avoiding round payment amounts, creating transactions with identical change output script types, and randomising change output positions. + - Equal output coinjoins are collaborative transactions involving multiple users. By breaking deterministic links these transactions create ambiguous transaction graphs. By involving multiple users, they defeat the CIOH. + - Payjoin transactions are also collaborative transactions. They involve the payer and payee in creating a transaction and have the same transaction fingerprint as a normal multi-input spend. Without an identifiable fingerprint, these transactions defeat the CIOH. + + + +# Article for privacy +- https://en.bitcoin.it/wiki/Privacy +- summary of how normal bitcoin users can improve their privacy. + - What is your threat model? + - Do not reuse addresses, Addresses should be shown to one entity to recieve moeny and never used again after the money from them is spent. + - avoid attaching email addresses, names, real live address and all kry when transacting. + - use your own full node when using a bitcoin wallet. + - broadcast onchain transactions over tor. + - use the lightening network. + - use a wallet that implements coinjoin. + - avoid creating change addresses, fund a lightening channgel iwth an entire utxo. + - use tails OS. +- Introduction + - the linkages between addresses made by transactions are often called the transaction graph. + - alone this information can't identify anyone because the addresses and transaction IDs are just random numbers. + - if any of the addresses in a transactions past or future can be tied to an actual identity it might be possible to work from that point and deduce who may own all the other addresses. + - you need forwards and backwards privacy. if you reveal your information when you receive the bitcoin then when you spend it you are not private. Or if you privatly receive the bitcoin but then give information when you spend it (give mailing address to recieve a good), then you can be linked to buying it. + - Multiple intepretations of a blockchain transaction + - how many possible ways is there to interpret a two input two output transaction? + - 9 + - Alice provides both inputs and pays 3 btc to Bob. Alice owns the 1 btc output (i.e. it is a change output). + - Alice provides both inputs and pays 1 btc to Bob, with 3 btc paid back to Alice as the change. + - Alice provides 1 btc input and Bob provides 3 btc input, Alice gets 1 btc output and Bob gets 3 btc output. This is a kind of CoinJoin transaction. + - Alice pays 2 btc to Bob. Alice provides 3 btc input, gets the 1 btc output; Bob provides 1 btc input and gets 3 btc. This would be a PayJoin transaction type. + - Alice pays 4 btc to Bob (but using two outputs for some reason). + - Fake transaction - Alice owns all inputs and outputs, and is simply moving coins between her own addresses. + - Alice pays Bob 3 btc and Carol 1 btc. This is a batched payment with no change address. + - Alice pays 3, Bob pays 1; Carol gets 3 btc and David gets 1 btc. This is some kind of CoinJoined batched payment with no change address. + - Alice and Bob pay 4 btc to Carol (but using two outputs). + - heuristics are used to excluded some of the possible 9 examples we said above. + + - a small data leak combined with other data leaks can be very dangerous, at first it might not be thought of as dangerous but then later on it can be. + - a public financial ledger is unheard of, we must do what we can to keep it private. +- Blockchain attacks on privacy + - wallet clusters aka addresses of a single entity are obtained and then later attempted to tie to a real identity. + - common input ownership heuristic. + - the purpose of coinjoin is to break this heuristic. + - change address detection + - Change address detection allows the adversary to cluster together newly created address, while the common-input-ownership heuristic and address reuse allows past addresses to be clustered. + - what are some common ways of change address detections. + - address reuse. + - a reused address is almost never a change address, making it known that the other one is. + + - wallet finger printing. + - what are ways of wallet finger printing? + - address formats. wallets typically only use one format type. + - script type, multi sig or signle sig script? + - if a wallet does or doesn't follow bip69 for ordering transctions. + - coin selection algorithms. + - unnessecary input heruistic. + - if a wallet puts in a utxo of 3 and 2, and there is an output of 4 and 1, the change must be 1, since if the payment was 1 then why would the user put in two inputs if it could have been covered by one. + + - sending to a different script type. + - equal output coin join. + - if the inputs are not equal, but some of the outputs are, you can tell which change outputs are related to which inputs. +- transaction graph heuristic. +- input amounts reveal wealth. + - if a user makes a small transactions but include a big utxo, they showed someone they are wealthy. +- a script type so unique it might narrow down who you are. or information about you. + - like what is a unchained address is so unique it shows a hacker that someone at unchained has 500 bitcoin, therefore they try to hack their unchained account. +- a merchant displaying addresses is dangerous, someone could start linking addresses to you and follwoing your wallet. +- dust attacks, recieve bitcoin hoping that a wallet includes it in a tx and then they can track you +- Methods for improving privacy (non-blockchain) + - obtain bitcoin ananomously. + - do not attach your name to the bitcoin, this is the single most important thing you can do. + - without this, the other heuristics do not provide much value. + - cash trades. + - mining bitcoin is the most private way. + - spending bitcoin anoymously. + - dont give your address when you spend bitcoin. + +- Methods for imrpove pricacy (blockchain) + - avoid address reuse. + - coin control + - change avoidance. + - not having a change output is great for privacy. + - create more than one change output. + - this breaks change output heiristics. + - coinjoin + - what makes a coinjoin effective? + - having more than one output with the same amount. + - PAyjoin + - when a person receiving a payment includes an input. + - helps break who is the sender and who is the receiver of the payment. + - silent payments +- Existing privacy solutions. + + +# Wasabi analysis +- https://archive.ph/LYXL0 +- more on anominity set calculation + - When remixing, the expectation is that you are taking the obtained anonymity set from the prior mixes (subtracting 1, since your 1 TXO is being consumed) and adding the anonymity set obtained in the second mix transaction. +If we were to remix our previously mentioned initial TXO in an additional CoinJoin transaction that obtained a further 50 anonymity set, we would expect a total score of 99 ( 50-1 + 50 ) +- due to how remixes are done this paper says + - Absent the implementation of a solution, users of Wasabi Wallet should be aware that the anonymity set of their postmix output may potentially be as low as the anonymity set provided by the last mix of the associated TXO. Users should act accordingly, depending on the specifics of the last mix and depending on their own threat model. +- overview of the issues in the paper + - Lack of Randomness: The main vulnerability identified is the deterministic coin selection algorithm used by Wasabi. When selecting unspent transaction outputs (TXOs) for mixing, the wallet does not introduce randomness. This allows an adversary to predict which TXOs will be included in future mixing rounds, thereby undermining the effectiveness of the CoinJoin process. + - Impact on Anonymity Set: The anonymity set—essentially the size of the crowd your coins are mixed with—is compromised. If an attacker can predict which TXOs will be mixed in a subsequent round, they can reduce the perceived anonymity of those outputs significantly. For instance, if an expected mix would yield an anonymity set of 99, the knowledge of which TXO will be remixed can drop this to just 50, severely diminishing privacy. + + +# Chainanaylse +- https://archive.ph/lzVXd +- important argument used by the chain anaylzse company + - The Bitcoin Fog cluster produced by Reactor is the result of 2 heuristics: an improved version of the Common Input Ownership Heuristic (i.e. all addresses used as inputs to a transaction are controlled by the same entity) and a second heuristic clustering addresses composing a peelchain (i.e. a long chain of transaction "peeling" small amounts of UTXOs for payments). + - another explanation of peel chain heuristic. + - https://www.linkedin.com/pulse/peel-chain-analysis-moneyflow-step-by-step-guide-bitquery-wvzkc/ + + + +# BITCOIN DESIGN ON PRIVACY +- https://bitcoin.design/guide/how-it-works/wallet-privacy/ -x + +# River +- https://river.com/learn/bitcoin-privacy-and-anonymity/ +- For chain analysis to be useful, it must be combined with some reliable starting data, such as the ownership of certain UTXOs or addresses. KYC/AML compliance by custodians and exchanges provide this starting data. If the ownership of a specific UTXO is known, when that UTXO is spent, chain analysis can attempt to determine whether it was sent to another party or it was sent back to the same owner. + +# Privacy Notes for live wallet +- have the ability for users or automatically mark transactions that most likely have PII associated with them. +- track change outputs and make sure that they are not used often, or if they are make it known that these two utxos are seen as related +- make sure addresses are not reused +- make sure utxos from sending to an exchange is not mixed with other utxos. + - this is because it is easy to identify the change. +- Give a utxo a score and then you can click into it and it will explain why based on a set of heuistics? +- not having randomized output positions for change is bad as well. if the wallet always uses the same output position for the change then that is a give a-way. +- use timing analysis to see if transactions always are happening on the same day or other timing analysis. +- be able to explain why a transaction was marked as not private. +- txs with multiple inputs and multiple outputs are noisy and not easily interpreted without special tooling or considerations. +- identify coinjoins. (this is good and makes the observer know they can't be sure how the inputs and outputs relate) +- identify paywalls / payjoins (this is good and helps great false clusters) + +- if a utxo is too big and you pay someone with it, and even worse if you attach an address to your payment, then that person has an incentive to wrench attack you, therefore highlight utxos that are too big, say over $10k? and say this is a big utxo. or maybe on a scale of size 10-100k. +- track utxos that have been through a coinjoin, these are highly deniable utxos +- a transaction that has the same input utxo as the recipient utxos is good, helps the change from the recipient. +- make sure that After using CoinJoin, a user should not combine the outputs of their CoinJoin transaction. This nullifies the privacy benefits of CoinJoin. +- ability to detect if it came from aan exchange or not, and ability to amend it if it did, and then also save that data so the user doesn't have to always add it. Would need to save it when you save the wallet. +- mayb a way to think of a score is if a wallet cluster was made of your waller from the outside what % of your utxos / transactions would be correctly clustered to you. + - the goal of chain analysis is to group addresses together. +- IPs are also analysized, hmm not sure how to incorportate that. way to differenciate your IP vs. VPN vs. tor being used for a transactions. +- maybe you should be able to mark a transaction as made public like for a donation or something. +- if you got your coins from mining then that is great for privacy +- think about did you receive the bitcoin privatly? Did you spend the bitcoin privately. Both forwards and backwards privacy is needed. +- reusing addresses is also bad because it makes it clear what the change was and what the payment was. +- https://checkbitcoinaddress.com/ can be used to check is an address has been posted online. +- be able to detect the unnessecary input heruistic. + - if a wallet puts in a utxo of 3 and 2, and there is an output of 4 and 1, the change must be 1, since if the payment was 1 then why would the user put in two inputs if it could have been covered by one. + - a way to not do this is to add more inputs so the change is actually more than the payment, this breaks the heuristic. +- identify the change from a partial equal output coinjoin with unequal inputs. make sure those change are never added back with the clean outputs. +- be able to mark utxos as clean or dirty, maybe clean is coinjoined or not. +- is there a way to identify dust attacks? +- make sure any data that is stored on the disk about the bitcoin wallet like addresses and keys etc, are encrpyted. +- maybe have the ability t oanaylsse a consolidation for privacy before you create the psbt? +- load in a psbt and tell the user how private this transaction is. + +What would the ideal utxo structure look like for privacy? +- utxos that arent too big. +- no links to exchanges +- utxos that are not linked together via exchanges +- various size utxos so that you can make payments of various sizes without needing to aggregate lots of utxos together. and potentially aggregating change together. + - as well when you pay someone they can now view the future transactions of the change knowing you are the owner of that utxo. +- some way to idenntify or hae a user mark if an address has been posted online publicly. that creates a starting point. +- should I include labeling in some way? + - change should be labeled with what it was used for. + - When payments are made, users should also get in the habit of labelling their change UTXOs. MAybe I should add this to the add highlighting that change utxos have not been labeled with what they have been used for. +- Give the user points for including any dummy transactions aka Ricochet — Adding Distance + - this should be easy to detect, detect when the user makes single input, two output txs to themselves. + + + +# Boltzmann and forward privacy. +- botlzman calculates the linkability of the inputs and the outputs of a bitcoin transaction +- https://github.com/Samourai-Wallet/boltzmann?tab=readme-ov-file + - This metrics can be applied to all bitcoin transactions but are specifically useful for qualifying the quality of a joint transaction + - This measures coinjoin entropy + - measure how many possible mapppings of inputs to outputs are possible given the values of the txos. + - no additional information is given to determine the probability. + - (I think this means like no other heuristics are used to weigh the reading of the transaction) + - Intrinsic entropy + - the value computed without any additional information, when the transaction is considered seprarted from the blockchain + - Actual entropy: the value taking into account additional information. + - this one matters the most to users. + - Max Entropy: + - it's the value associated to a perfect coinjoin transaction having a structure similar or close to the evalueted transaction + - What is the perfect coinjoin? + - a coinjoin with all inputs having the same amount and all outputs having the same amount. + - Wallet efficiency + - wallet efficient = intrinsic entropy - max entropy + - the efficiency of a wallet when it builds a coinjoin transaction. + - Blockchain efficientcy = Actual entropy - max entropy. + - Rule: Actual entropy of a bitcoin transaction is a dynamic value susceptible to decline over time. At best, it will stay constant, it will never increase. + - Limitations. + - This metric is susceptible to be tricked by specific patterns of transactions like steganographic transactions. + - What is a steganographic transaction? + - it aims to hide the real payment inside a fake payment. + - it involces the payee. + - basically a payjoin for the person receiving a payment + - A transaction with high entropy is likely to provide better privacy to the users. + - this fails to detect privacy leaks occuring at the lower lever of specific inputs/outputs. + + + - Implementations. + - brute force algo, no parallelization, no optimization. + - This is a reduction of the problem space by not including external information. + - inputs and outputs less then ten are needed due to the bad algo? + - did about 60-70% of bitcoin transactions. Done in 2015 + - 85% of transactions processed have a null entropy, they have inputs and outputs deterministically linked. + - 14% of the transactions processed have >= 1 (they are as good or better than ambigous transactions) + - 1% of the transactions process ahve >=1.5 (They are as good or better than most basix coin join transactions) + - these all don't ahve coinjoin like structure but they produce entropy like a coinjoin. + - chat gpt says a coinjoin 5 of 5 with equal input and output would be a 6.9 and a classic two input and two output of different sizes would be 0 entropy. + + - Questions + - propsed to add this to joinmarket from waxwing + - Questions from waxwing + - What does it mean for an input and an output to be linked? Common ownership of both utxos? valid payment, so different owners but a link between the two? + - Answers from LaurenMT + - a link between an input and an output captures the concept of an intentional finacial flow between the input and the output (taking into account the amount transferred to the output) but botlzmann doesn't care if a same entity controls a specific input and a specific output. + - Boltzman only cares about the links between the inputs and the outputs. it does not handle higher interpretations of who controls what. + - I don't think there is such a thing as a metric able to measure privacy. It is a fool's errand. BUT it is possible to define privacy metrics., a high score doesn't provide any certainty about your privacy but a low score might be a sign of potential privacy issues. + - Its recent introduction into Samourai Wallet (with a visual indicator of deterministic links) follows the same principle. It's used as a way to build awareness about the issue of deterministic links but it follows the principle that it should be used as a negative indicator (a high entropy doesn't give you any guaranteee but a low entropy isn't a good sign for your privacy). + - IMHO, the hardest part is all about the "education" of users. Almost all users expect tools providing a metrics telling them that their privacy is safe. The challenge for all of us is to change this mindset. + + - response from LaurenMT + - Boltzmann is focused on the question of "deterministic links" while the model you're discussing seems focused on the question "who controls what?". + - As a consequence, Boltzmann is "useless" for Payjoin/P2EP transactions (and almost all transactions with payments made between the senders). + - you model seems like an interesting approach which is complementary to Boltzmann. If we envision the bitcoin txs graph as a graph of interlinked utxos, your model has a focus on the ambiguity introduced by the nodes of the graph ("who controls which utxos?") while Boltzmann has a focus on the ambiguity introduced by the edges of the graph ("probability of a link between 2 utxos") + + + in terms of boltzman/bitcoin what is a deterministic link? + - chat gpt. + - + + + why is it good to have a high botlzman score + - A high Boltzmann score, which reflects greater entropy in a Bitcoin transaction, is beneficial for several reasons: + ### 1. Enhanced Privacy + - **Obfuscation of Input-Output Links**: A high entropy score indicates multiple valid mappings of inputs to outputs, making it difficult for observers to determine which inputs correspond to which outputs. This obfuscation helps protect users' identities and transaction details from analysis. + + ### 2. Resistance to Heuristic Attacks + - **Difficult to Trace**: Transactions with high entropy are less susceptible to heuristic methods (like the Merged Inputs Heuristic or CoinJoin Sudoku) that attempt to link inputs and outputs based on patterns. This makes it harder for analysts to de-anonymize users. + + ### 3. Improved User Anonymity + - **Better Protection Against Tracking**: With a higher Boltzmann score, the likelihood of being tracked through the blockchain is reduced. This is particularly important for users who want to maintain their financial privacy. + + ### 4. Encouragement of Best Practices + - **Transaction Design**: A high entropy score can guide users and developers to adopt better practices in transaction design, such as using varied input and output amounts or employing privacy-enhancing techniques like CoinJoin. + + ### 5. Increased Trust in the System + - **User Confidence**: Users may feel more secure using systems that promote high-entropy transactions, knowing their activities are less likely to be traced or linked back to them. + + ### Conclusion + In summary, a high Boltzmann score indicates greater uncertainty about the relationships between inputs and outputs in a transaction, enhancing privacy and protecting users from analysis. It fosters better transaction design practices and increases overall user confidence in the privacy of their transactions. + + + + + in the boltzman analysis do the amounts in the utxos matter? + - yes they are super important, if you put in an input of 1, and an input of 8, and there is an output of 7, and one of 2 that input can not be linked to taht output. + + + + + - ahh so it is all about tracking someone, and them having forward privacy. if you know the person before the transaction, which is the output that is theirs now? you don't know if you have a 5 input and 5 output tx of teh same amount. the person is now hiding in the crowd. + + +# Boltzman gist part 2 +- https://gist.github.com/LaurentMT/d361bca6dc52868573a2 + - didn't get anything from this article +# Botlzman gist part 3 + - https://gist.github.com/LaurentMT/e8644d5bc903f02613c6 + - potential attacks against the privacy provided by tools like coinjoin + - flaws in algo when new side-channel information is detected , (address clustering, etc) + - if several inputs are deterministically linked to a same output, addresses associated to these inputs can be clustered together + - you rely on other people in a coinjoin. this is a weakness. + - if you do a coinjoin that looks like + - input 1 (1 btc) + - input 2 (2 btc) + -------- + - output 1 (.8 btc) + - output 2 (1.2 btc) + - output 3 (.2 btc) + - output 4 (.8 btc) + --------- + + + - you have broken the link of two outputs from there inputs + - now these is plausibile deniability for output 1 and output 4, you don't know how they link back to the inputs. + - if one of those users was kyced, you now have split your odds from 100% to 50% + - but the problem is if the other user then combines their outputs 3 and 4 (.2 and .8), to make a 1 btc payment then it is obvious that the input of 2 btc in connected to output 1 and 2. and we have kyced them again. + - or even further down the line the user makes a payment with .8, gets change back and then combines that with output 3, they have now deannomoized themself again + - how to protect against this? + - enter multiple rounds of coinjoin + - do coinjoins that do not rely on just one other user. + + - finger printing + - conclusion + - one round of coinjoin is like no round at all + +# UI research, jam app for join market +- https://www.youtube.com/watch?v=FbyjG2upGO8 +- the docs have a good analogy about fruit -> jam, to annomozye your bitcoin + - I don't think the way the anology is presented in the app is that clear, but I like the idea of trying to move a non private amount utxo to more private as it moves along transactions + - they also have tags on their transactions + - for example one is cj-out, which means coin join out + - jam have 8-9-10 collaborators as the default + - the way join market works is that is takes inputs of various amounts, and then it creates many outputs with the same value + + +# UI research whirlpool in sparrow +- https://www.youtube.com/watch?v=6TcUY2yU41w + - badbank change is a label for the amount change sent back to a utxo used in a coinjoin whirlpool + - the badbank change is part of the premix transaction. + - sparrow breaks up outputs the into multiple "wallets". + - the badbank is now separate from the utxo in the whirlpool + - there are four "wallets", deposit, premix, postmix and badbank + - the ux is like tabs on the right side, and each time you switch it is as if it is a new wallet, aka different transactions, send tab, adress tab and utxos and settings tab on the left. + - money will move from the premix 500k utxo to, 5, 100k utxos in the post mix wallet +- In a single whirlpool you have a 1 out of 5 anominity set. + + +# UI research wasabi wallet +- https://www.youtube.com/watch?v=52pSd3I1nac +- allow you to choose coinjoin strategy. + - minimize cost, maximize speed, maxamize privacy. + - ability to add a hardware device alongside the hot wallet that is wasabi. +- they have a privacy progress bar. + - clicking into it gives a breakdown of the privacy of the utxos you own. + - lets you see the anomnty score. + - how does wasabi decide the anomity score? + - if the score is 22 then they have a 1 out of 22 chance of knowing which utxo ws mine. + - https://www.reddit.com/r/WasabiWallet/comments/194ko9i/how_does_the_anonymity_score_work/ + - The score on Wasabi 2.0 is calculated similarly to 1.0 such that each UTXO gains points based on the number of other coinjoin outputs in the transaction have the same matching value. The main difference from 1.0 is that you can receive multiple outputs of the same size, which divides your score between them. + + In regards to good privacy, it somewhat depends on the amount you are coinjoining since smaller users are able to hide their coins better than whales. If you are cautious about privacy, I recommend remixing your coins at least one time, which is about ~15 for your anonymity score target in most cases. I wouldn't recommend going above 50 anon score target unless you are coinjoining larger amounts. + + Coin control is no longer displayed because there's no more toxic change produced by coinjoins. Your entire balance is turned private, so there's no need to sort through labels to figure out which information you will share with others depending on how you construct your transaction. + - https://blog.wasabiwallet.io/what-is-the-difference-between-an-anonymity-set-and-an-anonymity-score/ + - TODO read this article. + - ui of the coins shows what type of address it is, this is important for not breaking the heuristic of the address type + - another nice ui feature is it shows you previous labels and allows you to attach them to other utxos + - another nice ui is it allows you to label the recipient wallet when you do a send. + - the manual coin control and send ui is nice. + - they kind of have a privacy analysizer themselves when a tx is created. + - they even have a recomendation with the ability to click it to imrpove privacy. + - https://primal.net/e/note1vnpkwaqdnw7mwhf9t3cpg5xes5ms5v8svwhv5gav4ep07jsrngksafsxvq + - Privacy score used in Wasabi is not an AnonSet, but an AnonScore. It is based on AnonSet but lowered if privacy harming behavior is detected, such as remixing with the same participants. Coordinators could also introduce a minimum amount of fresh bitcoin per rounds + - wabsi sabi is weakest for the very largest wale in the transaction + - podcast with some insight on the ui + - https://www.youtube.com/watch?v=v952Fd1vmOs&ab_channel=BitcoinTakeover + - 2.0 tried to hide utxos. where as one had great coin control. This was back tracked a bit though. + + -annominity score vs annominity set + - https://blog.wasabiwallet.io/what-is-the-difference-between-an-anonymity-set-and-an-anonymity-score/ + - wasabi 1.0 had an annominity set that it used that it gave to utxos. + - his measurement was called an “anonymity set,” which is equal to the number of peers in your coinjoin transactions that share an output of an identical value + - An anonymity set is a number that is equal to the size of the group you’re hiding in. For example, if you participate in a coinjoin round with 50 different peers in which each provides an input and each gets 1 coinjoined output of equal amounts, the anonymity set for each of the latter would be 50. On the other hand, the anonymity score extends the definition of the previous term by considering edge cases to give a more precise and conservative definition of quantifiable privacy guarantee. + - annominity set is equal to the size of the group you are hiding in. how many other users have the same output utxo as you? + - if you are part of multiple coin join rounds your annominity set increases becausae it accumulates peers each time, increasing the size you are hiding in. + - in wasabi 2.0 the anominity score is always equal to or lower than your annoniminity set. + - the annoniminity score comes from the coordinator, since they have more information than an outside observer. + - the anonymity score is client-specific since it takes into consideration the existing privacy of inputs and number of outputs a client registered in a given coinjoin round. + - The first edge case is if you have multiple outputs of the same denomination in a coinjoin transaction, your anonymity score becomes lower than if you only had one. + - what about mixing coins + - dont mix postmix (or any post mix tx change) and non mixed. + - dont mix post mixes (or any post mix tx change) from different rounds. + - if you mix post mixed from the same round that is better but it reduces you annominity set by two. + +- one thing that stinks is that if you send a tx to a ne wwallet after it has been coinjoined, then it doesn't give it a high anomity score because it didn't just come from a coinjoin it looks like you just received your first payment. + - would be nice if you have the aility to mark it as coming from a coinjoin and then it can mark the anomity set from the previous tx. + - actually I do think they can recognize this if you select the wallet as the receive wallet and don't do a manual send. + + + +- hmm should I just analyse collaborative transactions vs regular ones. +- hmm maybe a collaborative custody buddy. + - helps keep clean utxos from being combined with dirty ones. + + + +# GREAT IDEA +- transaction privacy analysis based on selected privacy policies. +- a user will be able to load in a new psbt or select one of their previous transactions which will all be laid out for them, and they will be able to select which privacy policies they want to analysise the transaction for and then hit analyse and then it will result in is the policy was passed or not. +- another idea would not just be transaction analysis but a transaction builder to follow these, that seems more advanced though. + - but if I can analyse them I should be able to build them. +- Will I need to be able to view indivudal transactions like sparrow or mempool? + - showing the inputs and the outputs and the amounts associated with them, and the address they came from and are going to? +- What are a series of privacy policies a user can choose? +- You should be able to browse policies and read about what they are and why it is good. +- some of this may require labeling from the user before hand. Like is this utxo kyced or not. +- TODO: come up with policies and an explanation of what they are and why it is good. + - # Selectable privacy policies + - number of consecutive coinjoin rounds that a utxo came from. + - choose your anoominity set, aka how many txs should look like yours (aka equal output) in a tx. + - should be able to calculate based off of multiple transactions + - No change. + - no small change. + - you probably are better off with no small change for privacy than change with a small amount. + - no reused addresses + - another reason why address reuse is bad is because anyone can look up my address and see my wallet balance. as well they could see every tx i hae made or received + - reveal least amount of wealth + - maybe even have the user able to set what they consider an amount that is okay to show that they own above the payment. + - reveal least amount of past tx history, aka don't combine utxos, especially not change outputs. + - dont use utxos that has come from many other transactions you were involved in. if you use a 1 btc to pay for .005, and then use the change to pay again and again again, everytime you use it it may be obvious your past transactions + - no wallet clustering (aka no combining change if possible) + - dont mix post mixes (or any post mix tx change) from different rounds. + - if you mix post mixed from the same round that is better but it reduces you annominity set by two. + - dont mix postmix (or any post mix tx change) and non mixed. + - a more general one like "no traceable change" which includes sub field. + - make sure no address reuse + - no round number payments. + - no same script type for the inputs and only one output. + - largest output amount heuristic (this is weak) + - spending to an exchange, it is obvious you are the change + - break the unnessesary input heuristic. + - more than one change output. + - change output should be of similar size of payment + - make sure dust attacked coins are not spent. + - break the common input odwnership heuristic. + - aka this would require an input from someone else. + - aka a payjoin + - break volume or other pattern recognition analysise. + - the change should not always be in the same output position. + - timing analysis, make sure you are not always receiving transactions on the same day. + - make sure do not spend utxos are not spent + - I would need the ability to mark them as do not spend though. + - must require dummy transactions before hitting recipient. + - aka include riochets. + - break peel chain heuristic + - its “peel chain” heuristic assumes that unspent Bitcoins are linked along a chain where the bigger transaction is the spender keeping their “change. + - I am not 100% sure about this one. + - be able to mark a transaction as publicly known, and then mark don't spend / include from publicly known addresses. + - no kyc coins. + - ability to mark kyced vs non kyced vs. not labeled. + - ability to label, previously coinjoined. + + + - explanations should include, to break this heuristic you must have a number of outputs that have an equal value. to break this heuristic, you must not include utxos with large amounts. you must use the smallest amount utxos available. + + +- ability to differenciate between what coins are used and what the result fo the transaction is? +- should I just do an annominity set nlayization like wasabi. + - yeah I love this for each utxo. +- it is good to analyse past transaction because this may help highlight what utxos (from previous txs receiving or spendign and getting hcange need to be coinjoined.) +- ANOTHER IDEA FOR THE APP IS SELECTING THESE requirement boxes and an amount AND THEN HIGHLINGHTING WHICH combination of utxos would be usable for this type of transaction. + +# what labels are available in live wallet to attach to utxos. +- do not spend +- coming from coinjoin from other wallet I own. +- kyced +- no-kyced. +- maybe custom label +- do I need the dust attack label? + + +# Initial thoughts on how v1 UI will look +- Do I need to distinguish between transactions the user makes vs. ones where they receive bitcoin? +- Receiving bitcoin is pretty simple privacy. You just create a new address. But for a utxo, you want annominity set and you want no reused addresses. +- For sending there are a ton more privacy metrics. +- I need tabs, privacy / efficiency + - There is a little side bar padding already, I could add two little buttons there, one with a wallet icon one with a private icon to switch between the two modes. + - Or I could add it right in the middle of the top header. +- Privacy section tabs + - If needed I could have sparrow like tabs on the side for transactions and utxos. And a section for new + - New (or call it psbts) hm maybe just “preview” + - New will be where you put psbts. And then you can analyze them. + - Maybe it is called preview because you are previewing or analyzing a tx before it is sent/complete. + - Utxos tab + - This is where you can mark utxos, label them as kya or no kyc. + - Should be able to mark SPENT UTXOS, therethre should be able to toggle showing STXOS so you can label those as well. + - Also will be in a table where it has the amanimity set. + - Will also be red if it has a reused address. + - Able to mark do not spend on utxos. + - Able to mark previously conjoined. + - The order should be transactions, utxos, new. This way it goes from most complete to least complete. + - The ui for analyzing a previous tx and a psbt will be the same ui. Selecting how you want to analysis, and then hit an analyst button, and then see the same ui output for how well the tx did and suggestions for it to be improved / could have been improved. + - have a page that lists all privacy metrics and explains them all. + - flow, pick your transaction (or psbt), hit next or some button, pick metrics you want to analyze by, hit analyze button, view output. Maybe ability to save it? +Eventually add a create tab which will go through a similar flow of, select amount, select privacy metrics, then it will pick which utxos you should include. And maybe even create that psbt for you? +- maybe a feature showing most likely cluster? + + +# TODO NEXT STEPS +- build a display of all transactions +- build a utxo annonominity set viewer for all utxos and for individual ones. + + + +Kruw on bitcoin privacy +- https://www.youtube.com/watch?v=v952Fd1vmOs&ab_channel=BitcoinTakeover +- I guess wabi sabi protocol the output size is not known until after the inputs are collected. + + +- good video showing post mix fail from wasabi + - https://www.youtube.com/watch?v=alcLdBsoDDg From 71af67af0da3e1ec16d069dc3e34edd777049a86 Mon Sep 17 00:00:00 2001 From: Joseph Wyman Date: Thu, 17 Oct 2024 06:11:31 -0400 Subject: [PATCH 02/85] add initial privacy tab layour --- src/app/pages/Home.tsx | 207 +++++++++++++++++++++----------------- src/app/pages/Privacy.tsx | 53 ++++++++++ 2 files changed, 168 insertions(+), 92 deletions(-) create mode 100644 src/app/pages/Privacy.tsx diff --git a/src/app/pages/Home.tsx b/src/app/pages/Home.tsx index 9f7b964d..2e5dbc2a 100644 --- a/src/app/pages/Home.tsx +++ b/src/app/pages/Home.tsx @@ -32,6 +32,7 @@ import { Wallet, WalletConfigs } from '../types/wallet'; import { useGetBtcPrice } from '../hooks/price'; import { Pages } from '../../renderer/pages'; import { ScriptTypes } from '../types/scriptTypes'; +import { Privacy } from './Privacy'; export type ScaleOption = { value: string; @@ -44,6 +45,10 @@ export enum TxMode { BATCH = 'BATCH', CONSOLIDATE = 'CONSOLIDATE', } +export enum DisplayType { + PRIVACY = 'PRIVACY', + EFFICENCY = 'EFFICIENCY', +} function Home() { const getBalanceQueryRequest = useGetBalance(); const navigate = useNavigate(); @@ -62,6 +67,8 @@ function Home() { const [txMode, setTxMode] = useState(TxMode.SINGLE); const isCreateBatchTx = txMode === TxMode.BATCH; + const [displayType, setDisplayType] = useState(DisplayType.EFFICIENCY); + const location = useLocation(); const { numberOfXpubs, signaturesNeeded } = location.state as { numberOfXpubs: number; @@ -451,6 +458,18 @@ function Home() {
+ { + const selectedValue = + value === DisplayType.EFFICENCY + ? DisplayType.EFFICENCY + : DisplayType.PRIVACY; + setDisplayType(selectedValue); + }} + data={[DisplayType.EFFICENCY, DisplayType.PRIVACY]} + />

Balance:{' '} @@ -477,110 +496,114 @@ function Home() {

-
-
- + ) : ( +
+
-
-

- Consolidation Tx Fee Rate (sat/vB) + +
+

+ Consolidation Tx Fee Rate (sat/vB) +

+ +
+ +
+

+ Future Fee Environment (sat/vB)

+
+
+
+ + + + Fee rate: {feeRate.toLocaleString()} sat/vB + +
+
+
+
+ +
+

BTC Price

- -
-

- Future Fee Environment (sat/vB) -

-
-
-
- - - - Fee rate: {feeRate.toLocaleString()} sat/vB - -
-
-
-
-

BTC Price

- -
+

- - -
+ )}
); } diff --git a/src/app/pages/Privacy.tsx b/src/app/pages/Privacy.tsx new file mode 100644 index 00000000..07572e1d --- /dev/null +++ b/src/app/pages/Privacy.tsx @@ -0,0 +1,53 @@ +import { Tabs, rem } from '@mantine/core'; +import { IconCoins, IconArrowsDownUp, IconEye } from '@tabler/icons-react'; +export const Privacy = () => { + const iconStyle = { width: rem(22), height: rem(22) }; + enum PrivacyTabs { + UTXOS_STXOS = 'utxos_stxos', + TRANSACTIONS = 'transactions', + PREVIEW = 'preview', + } + return ( +
+ + + } + > +
UTXOS
STXOS
+ + } + > + Transactions + + } + > + Preview + + + + + My utxos and stxos + + + + My transactions + + + My preview area + +
+ ); +}; From c49accf0480fda98d742971c2dc538d6be0b1cc4 Mon Sep 17 00:00:00 2001 From: Joseph Wyman Date: Fri, 18 Oct 2024 09:09:36 -0400 Subject: [PATCH 03/85] add initial transactions fe -> BE -> electrum server dance --- backend/requirements.txt | 1 + backend/src/controllers/utxos.py | 32 ++++ backend/src/services/wallet/wallet.py | 99 ++++++++++- src/app/api/api.ts | 11 ++ src/app/components/privacy/txosTable.tsx | 203 +++++++++++++++++++++++ src/app/hooks/utxos.ts | 7 + src/app/pages/Privacy.tsx | 5 + 7 files changed, 357 insertions(+), 1 deletion(-) create mode 100644 src/app/components/privacy/txosTable.tsx diff --git a/backend/requirements.txt b/backend/requirements.txt index dd34a990..27f6a18b 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -16,3 +16,4 @@ chardet charset-normalizer==2.1.0 hwi===3.0.0 cryptography==42.0.8 +bitcoinlib==0.6.15 diff --git a/backend/src/controllers/utxos.py b/backend/src/controllers/utxos.py index ec5ed085..817713b0 100644 --- a/backend/src/controllers/utxos.py +++ b/backend/src/controllers/utxos.py @@ -119,3 +119,35 @@ def get_utxos( except Exception as e: LOGGER.error("error getting utxos", error=e) return SimpleErrorResponse(message="error getting utxos").model_dump() + + +# TODO put this url somehwere else +@utxo_page.route("/transactions") +@inject +def get_txos( + wallet_service: WalletService = Provide[ServiceContainer.wallet_service], +): + """ + Get all utxos and spend txos in the wallet. + """ + try: + txos = wallet_service.get_all_transactions() + + # TODO type response + return txos + # GetAllUtxosResponseDto.model_validate( + # dict( + # utxos=[ + # { + # "txid": utxo.outpoint.txid, + # "vout": utxo.outpoint.vout, + # "amount": utxo.txout.value, + # } + # for utxo in utxos + # ] + # ) + # ).model_dump() + # + except Exception as e: + LOGGER.error("error getting txos", error=e) + return SimpleErrorResponse(message="error getting txos").model_dump() diff --git a/backend/src/services/wallet/wallet.py b/backend/src/services/wallet/wallet.py index e62082eb..a7b3255f 100644 --- a/backend/src/services/wallet/wallet.py +++ b/backend/src/services/wallet/wallet.py @@ -1,6 +1,8 @@ +import json +import socket from dataclasses import dataclass import bdkpython as bdk -from typing import Literal, Optional, cast, List +from typing import Any, Literal, Optional, cast, List from src.database import DB @@ -20,6 +22,7 @@ from dependency_injector.wiring import inject import structlog +from bitcoinlib.transactions import Transaction LOGGER = structlog.get_logger() @@ -38,6 +41,56 @@ class GetFeeEstimateForUtxoResponseType: psbt: Optional[bdk.PartiallySignedTransaction] +def electrum_request( + url: str, + port: int, + method: str, + # TODO improve typing here + params: None | List[str | bool], + request_id: Optional[int] = 1, +): + if params is None: + params = [] + # Create a TCP socket + try: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.connect((url, port)) + + request = json.dumps( + {"jsonrpc": "2.0", "id": request_id, "method": method, "params": params} + ) + + # Send the request + s.sendall((request + "\n").encode("utf-8")) + + # Receive the response + response = b"" + while True: + part = s.recv(4096) + response += part + if len(part) < 4096: + break + + # Decode and parse the JSON response + response_data = json.loads(response.decode("utf-8").strip()) + return response_data + # TODO improve logging and error handling + except socket.error as e: + print(f"Socket error: {e}") + return {"error": str(e)} + except json.JSONDecodeError as e: + print(f"JSON decode error: {e}") + return {"error": "Invalid JSON response"} + except Exception as e: + print(f"An error occurred: {e}") + return {"error": str(e)} + + +def parse_electrum_url(electrum_url: str) -> tuple[Optional[str], Optional[str]]: + url, port = electrum_url.split(":") + return url, port + + class WalletService: """Initiate a wallet using the bdk library and offer various methods to interact with it. The wallet will use an electrum server to obtain data around it's transaction history and current utxos. @@ -136,12 +189,14 @@ def connect_wallet( ) db_config = bdk.DatabaseConfig.MEMORY() + cls.url = electrum_url blockchain_config = bdk.BlockchainConfig.ELECTRUM( bdk.ElectrumConfig(electrum_url, None, 2, 30, stop_gap, True) ) blockchain = bdk.Blockchain(blockchain_config) + cls.blockchain = blockchain wallet = bdk.Wallet( descriptor=wallet_descriptor, @@ -255,6 +310,48 @@ def get_all_utxos(self) -> List[bdk.LocalUtxo]: utxos = self.wallet.list_unspent() return utxos + # TODO add return typing + def get_all_transactions( + self, + ) -> List[Any]: + """Get all transactions for the current wallet.""" + wallet_details = Wallet.get_current_wallet() + if self.wallet is None or wallet_details is None: + # TODO should I throw an error here? + return [] + + electrum_url = wallet_details.electrum_url + + if electrum_url is None: + # TODO should I throw an error here? + return [] + + url, port = parse_electrum_url(electrum_url) + if url is None or port is None: + # TODO should I throw an error here? + return [] + + transactions = self.wallet.list_transactions(False) + + all_tx_details = [] + + for index, transaction in enumerate(transactions): + result = electrum_request( + url, + int(port), + "blockchain.transaction.get", + [transaction.txid, False], + index, + ) + + # Decode the transaction + transaction = Transaction.parse(result["result"], strict=True) + # TODO add logging? + + # TODO return as a dataclass? + all_tx_details.append(transaction.as_dict()) + return all_tx_details + def get_utxos_info(self, utxos_wanted: List[bdk.OutPoint]) -> List[bdk.LocalUtxo]: """For a given set of txids and the vout pointing to a utxo, return the utxos""" existing_utxos = cast(List[bdk.LocalUtxo], self.get_all_utxos()) diff --git a/src/app/api/api.ts b/src/app/api/api.ts index 5fb2bba2..17877c0a 100644 --- a/src/app/api/api.ts +++ b/src/app/api/api.ts @@ -54,6 +54,17 @@ export class ApiClient { return data as GetUtxosResponseType; } + + static async getTransactions() { + const response = + await fetchHandler(`${configs.backendServerBaseUrl}/utxos/transactions +`); + + const data = await response.json(); + + // TODO add a type + return data; //as GetUtxosResponseType; + } static async createTxFeeEstimation( utxos: UtxoRequestParam[], feeRate: number = 1, diff --git a/src/app/components/privacy/txosTable.tsx b/src/app/components/privacy/txosTable.tsx new file mode 100644 index 00000000..f5298603 --- /dev/null +++ b/src/app/components/privacy/txosTable.tsx @@ -0,0 +1,203 @@ +import React, { useMemo } from 'react'; + +import { createTheme, ThemeProvider } from '@mui/material'; +import { + MaterialReactTable, + useMaterialReactTable, +} from 'material-react-table'; + +import { CopyButton, rem, Tooltip, ActionIcon } from '@mantine/core'; +import { IconCheck, IconCopy } from '@tabler/icons-react'; + +const sectionColor = 'rgb(1, 67, 97)'; + +type TxosTableProps = {}; + +export const TxosTable = ({}: TxosTableProps) => { + const columns = useMemo(() => { + const defaultColumns = [ + { + header: 'Txid', + size: 100, + accessorKey: 'txid', + Cell: ({ row }: { row: any }) => { + const prefix = row.original.txid.substring(0, 4); + const suffix = row.original.txid.substring( + row.original.txid.length - 4, + ); + const abrv = `${prefix}....${suffix}`; + return ( +
+ +

{abrv}

+ + + {({ copied, copy }) => ( + + + {copied ? ( + + ) : ( + + )} + + + )} + +
+ ); + }, + }, + { + header: 'Amount', + accessorKey: 'amount', + size: 100, + Cell: ({ row }: { row: any }) => { + return ( +
+

amount

+
+ ); + }, + }, + { + header: '$ Amount', + accessorKey: 'amountUSD', + size: 100, + Cell: ({ row }: { row: any }) => { + return ( +
+

{'amountUSDDisplay'}

+
+ ); + }, + }, + { + header: '~ Fee %', + accessorKey: 'selfCost', + size: 100, + Cell: ({ row }: { row: any }) => { + return ( +
+

{'hi'}

+
+ ); + }, + }, + { + header: '$ Fee', + accessorKey: 'feeUSD', + size: 100, + Cell: () => { + return ( +
+

{'amountUSDDisplay'}

+
+ ); + }, + }, + ]; + + return defaultColumns; + }, []); + + const table = useMaterialReactTable({ + columns, + data: [], + enableRowSelection: false, + enableDensityToggle: false, + enableFullScreenToggle: false, + enableFilters: false, + enableColumnFilters: false, + enableColumnActions: false, + enableHiding: false, + enablePagination: false, + enableTableFooter: false, + enableBottomToolbar: false, + muiTableContainerProps: { + className: 'overflow-auto transition-all duration-300 ease-in-out', + style: { maxHeight: false ? '24rem' : '30rem' }, + }, + enableStickyHeader: true, + enableTopToolbar: true, + positionToolbarAlertBanner: 'none', + positionToolbarDropZone: 'top', + renderTopToolbarCustomActions: ({ table }) => { + return ( +
+

+ {'mock title'} + (utxos) +

+
+ ); + }, + muiSelectCheckboxProps: { + color: 'primary', + }, + initialState: { + sorting: [ + { + id: 'amount', + desc: false, + }, + ], + }, + + // @ts-ignore + muiTableBodyRowProps: { classes: { root: { after: 'bg-green-100' } } }, + muiTableBodyCellProps: ({ row }) => { + return { + //style: { backgroundColor: color }, + }; + }, + + getRowId: (originalRow) => { + return originalRow.txid; + }, + }); + + return ( +
+ + + +
+ ); +}; diff --git a/src/app/hooks/utxos.ts b/src/app/hooks/utxos.ts index c1158d48..9779c6b2 100644 --- a/src/app/hooks/utxos.ts +++ b/src/app/hooks/utxos.ts @@ -5,6 +5,7 @@ import { useMutation, useQuery } from 'react-query'; export const uxtoQueryKeys = { getBalance: ['getBalance'], getUtxos: ['getUtxos'], + getTransactions: ['getTransactions'], getCurrentFees: ['getCurrentFees'], }; @@ -19,6 +20,12 @@ export function useGetUtxos() { }); } +export function useGetTransactions() { + return useQuery(uxtoQueryKeys.getTransactions, () => ApiClient.getTxos(), { + refetchOnWindowFocus: true, + }); +} + export function useCreateTxFeeEstimate( utxos: UtxoRequestParam[], feeRate: number, diff --git a/src/app/pages/Privacy.tsx b/src/app/pages/Privacy.tsx index 07572e1d..64c6e4dd 100644 --- a/src/app/pages/Privacy.tsx +++ b/src/app/pages/Privacy.tsx @@ -1,5 +1,7 @@ import { Tabs, rem } from '@mantine/core'; import { IconCoins, IconArrowsDownUp, IconEye } from '@tabler/icons-react'; +import { useGetTransactions } from '../hooks/utxos'; + export const Privacy = () => { const iconStyle = { width: rem(22), height: rem(22) }; enum PrivacyTabs { @@ -7,6 +9,9 @@ export const Privacy = () => { TRANSACTIONS = 'transactions', PREVIEW = 'preview', } + + const transactionsResponse = useGetTransactions(); + console.log('transactionsResponse', transactionsResponse.data); return (
Date: Fri, 18 Oct 2024 09:14:27 -0400 Subject: [PATCH 04/85] add --- NOTES.md | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/NOTES.md b/NOTES.md index 856e3380..33570c68 100644 --- a/NOTES.md +++ b/NOTES.md @@ -868,3 +868,38 @@ Kruw on bitcoin privacy - good video showing post mix fail from wasabi - https://www.youtube.com/watch?v=alcLdBsoDDg + + + + + +# Electrum notes +- When I was trying to get all the information about the transactions in a wallet I came across an issue, bdk python has not implemented the get_tx electrum request. +- They did implement it for esplora though +- https://github.com/bitcoindevkit/bdk-ffi/pull/598/files +- https://github.com/bitcoindevkit/bdk-ffi/pull/601 +- https://github.com/bitcoindevkit/bdk-ffi/issues/596 +- Maybe I can add it to electrum + - https://github.com/bitcoindevkit/bdk-ffi/blob/master/bdk-ffi/src/electrum.rs + +- I ended up adding it myself by implemnting my own request to electrum + +Electrum client is in python +- https://electrum.readthedocs.io/en/latest/console.html +- Maybe I can grab code from there? +- Electrum server code + - https://github.com/spesmilo/electrum-server +- Electrum server docs + - https://electrumx-spesmilo.readthedocs.io/en/latest/HOWTO.html#howto + +- But it turns out electrum only returned the raw hex data not nicely formatted data so I had to parse it, to do that I needed to import another bitcoin library +- Bitcoin lib +- https://github.com/1200wd/bitcoinlib +- https://bitcoinlib.readthedocs.io/en/latest/source/bitcoinlib.transactions.html#bitcoinlib.transactions.Transaction +- peter todd also had one but it wasn't well documented like the one I am using + - https://pypi.org/project/python-bitcoinlib/#history + - https://github.com/petertodd/python-bitcoinlib + + + + From 57b59e699a91704abac6f969b3f0ace796f0ff16 Mon Sep 17 00:00:00 2001 From: Joseph Wyman Date: Sat, 19 Oct 2024 11:39:04 -0400 Subject: [PATCH 05/85] improve get transactions typing, error handling, logging and file structure --- backend/src/api/__init__.py | 1 + backend/src/api/electrum.py | 100 ++++++++++++++++++ backend/src/app.py | 2 + backend/src/controllers/__init__.py | 1 + backend/src/controllers/transactions.py | 35 ++++++ backend/src/controllers/utxos.py | 32 ------ backend/src/my_types/__init__.py | 1 + .../my_types/controller_types/utxos_dtos.py | 75 ++++++++++++- backend/src/services/wallet/wallet.py | 90 ++++------------ src/app/api/api.ts | 6 +- src/app/api/types.ts | 67 ++++++++++++ src/app/hooks/utxos.ts | 2 +- 12 files changed, 306 insertions(+), 106 deletions(-) create mode 100644 backend/src/api/electrum.py create mode 100644 backend/src/controllers/transactions.py diff --git a/backend/src/api/__init__.py b/backend/src/api/__init__.py index f80eeb29..bb7717e9 100644 --- a/backend/src/api/__init__.py +++ b/backend/src/api/__init__.py @@ -1 +1,2 @@ from src.api.fees import get_fees +from src.api.electrum import electrum_request, parse_electrum_url, ElectrumMethod diff --git a/backend/src/api/electrum.py b/backend/src/api/electrum.py new file mode 100644 index 00000000..eb6198e1 --- /dev/null +++ b/backend/src/api/electrum.py @@ -0,0 +1,100 @@ +from enum import Enum +from typing import List, Optional, Literal +from dataclasses import dataclass +from bitcoinlib.transactions import Transaction +import structlog +import json +import socket + +LOGGER = structlog.get_logger() + + +def parse_electrum_url(electrum_url: str) -> tuple[Optional[str], Optional[str]]: + url, port = electrum_url.split(":") + return url, port + + +class ElectrumMethod(Enum): + GET_TRANSACTIONS = "blockchain.transaction.get" + + +@dataclass +class GetTransactionsRequestParams: + txid: str + verbose: bool + + def create_params_list(self) -> List[str | bool]: + return [self.txid, self.verbose] + + +GetTransactionsResponse = Transaction + +ALL_UTXOS_REQUEST_PARAMS = GetTransactionsRequestParams +ElectrumRawResponses = dict[str, str] +ElectrumDataResponses = GetTransactionsResponse + + +@dataclass +class ElectrumResponse: + status: Literal["success", "error"] + data: Optional[ElectrumDataResponses] + + +def electrum_request( + url: str, + port: int, + electrum_method: ElectrumMethod, + params: Optional[ALL_UTXOS_REQUEST_PARAMS], + request_id: Optional[int] = 1, +) -> ElectrumResponse: + # Create a TCP socket + try: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.connect((url, port)) + + request = json.dumps( + { + "jsonrpc": "2.0", + "id": request_id, + "method": electrum_method.value, + "params": params.create_params_list() if params else [], + } + ) + + LOGGER.info(f"Sending electrum request: {request}") + # Send the request + s.sendall((request + "\n").encode("utf-8")) + + # Receive the response + response = b"" + while True: + part = s.recv(4096) + response += part + if len(part) < 4096: + break + + # Decode and parse the JSON response + raw_response_data = json.loads(response.decode("utf-8").strip()) + + # Decode the transaction + response_data = handle_raw_electrum_response( + electrum_method, raw_response_data + ) + return ElectrumResponse(status="success", data=response_data) + except socket.error as e: + LOGGER.error(f"Socket error: {e}") + return ElectrumResponse(status="error", data=None) + except json.JSONDecodeError as e: + LOGGER.error(f"JSON decode error: {e}") + return ElectrumResponse(status="error", data=None) + except Exception as e: + LOGGER.error(f"An error occurred: {e}") + return ElectrumResponse(status="error", data=None) + + +def handle_raw_electrum_response( + electrum_method: ElectrumMethod, raw_response: dict +) -> ElectrumDataResponses: + if electrum_method == ElectrumMethod.GET_TRANSACTIONS: + transaction = Transaction.parse(raw_response["result"], strict=True) + return transaction diff --git a/backend/src/app.py b/backend/src/app.py index c3ef7f41..a87e215d 100644 --- a/backend/src/app.py +++ b/backend/src/app.py @@ -19,6 +19,7 @@ def create_app(cls) -> Flask: from src.controllers import ( balance_page, utxo_page, + transactions_page, fees_api, wallet_api, health_check_api, @@ -45,6 +46,7 @@ def create_app(cls) -> Flask: cls.app.container = container cls.app.register_blueprint(balance_page) cls.app.register_blueprint(utxo_page) + cls.app.register_blueprint(transactions_page) cls.app.register_blueprint(fees_api) cls.app.register_blueprint(wallet_api) cls.app.register_blueprint(health_check_api) diff --git a/backend/src/controllers/__init__.py b/backend/src/controllers/__init__.py index a33877f8..89737eba 100644 --- a/backend/src/controllers/__init__.py +++ b/backend/src/controllers/__init__.py @@ -1,5 +1,6 @@ from .balance import balance_page from .utxos import utxo_page +from .transactions import transactions_page from .fees import fees_api from .wallet import wallet_api diff --git a/backend/src/controllers/transactions.py b/backend/src/controllers/transactions.py new file mode 100644 index 00000000..71b81b63 --- /dev/null +++ b/backend/src/controllers/transactions.py @@ -0,0 +1,35 @@ +from flask import Blueprint + +from src.services import WalletService +from dependency_injector.wiring import inject, Provide +from src.containers.service_container import ServiceContainer +import structlog + +from src.my_types import ( + GetAllTransactionsResponseDto, +) +from src.my_types.controller_types.generic_response_types import SimpleErrorResponse + +transactions_page = Blueprint("get_transactions", __name__, url_prefix="/transactions") + +LOGGER = structlog.get_logger() + + +@transactions_page.route("/") +@inject +def get_txos( + wallet_service: WalletService = Provide[ServiceContainer.wallet_service], +): + """ + Get all transactions in the wallet. + """ + try: + transactions = wallet_service.get_all_transactions() + + return GetAllTransactionsResponseDto.model_validate( + dict(transactions=[transaction.as_dict() for transaction in transactions]) + ).model_dump() + + except Exception as e: + LOGGER.error("error getting txos", error=e) + return SimpleErrorResponse(message="error getting txos").model_dump() diff --git a/backend/src/controllers/utxos.py b/backend/src/controllers/utxos.py index 817713b0..ec5ed085 100644 --- a/backend/src/controllers/utxos.py +++ b/backend/src/controllers/utxos.py @@ -119,35 +119,3 @@ def get_utxos( except Exception as e: LOGGER.error("error getting utxos", error=e) return SimpleErrorResponse(message="error getting utxos").model_dump() - - -# TODO put this url somehwere else -@utxo_page.route("/transactions") -@inject -def get_txos( - wallet_service: WalletService = Provide[ServiceContainer.wallet_service], -): - """ - Get all utxos and spend txos in the wallet. - """ - try: - txos = wallet_service.get_all_transactions() - - # TODO type response - return txos - # GetAllUtxosResponseDto.model_validate( - # dict( - # utxos=[ - # { - # "txid": utxo.outpoint.txid, - # "vout": utxo.outpoint.vout, - # "amount": utxo.txout.value, - # } - # for utxo in utxos - # ] - # ) - # ).model_dump() - # - except Exception as e: - LOGGER.error("error getting txos", error=e) - return SimpleErrorResponse(message="error getting txos").model_dump() diff --git a/backend/src/my_types/__init__.py b/backend/src/my_types/__init__.py index f9a16be3..8e102267 100644 --- a/backend/src/my_types/__init__.py +++ b/backend/src/my_types/__init__.py @@ -6,6 +6,7 @@ GetUtxosResponseDto, GetUtxosErrorResponseDto, GetAllUtxosResponseDto, + GetAllTransactionsResponseDto, ) from src.my_types.controller_types.fees_dtos import GetCurrentFeesResponseDto diff --git a/backend/src/my_types/controller_types/utxos_dtos.py b/backend/src/my_types/controller_types/utxos_dtos.py index 71fe98cc..a5e64d55 100644 --- a/backend/src/my_types/controller_types/utxos_dtos.py +++ b/backend/src/my_types/controller_types/utxos_dtos.py @@ -1,4 +1,4 @@ -from typing import Optional +from typing import Optional, List from pydantic import BaseModel, field_validator, Field import structlog @@ -10,6 +10,75 @@ class TransactionDto(BaseModel): vout: int +class InputDto(BaseModel): + index_n: int + prev_txid: str + output_n: int + script_type: str # sig_pubkey + address: str + value: int + public_keys: str + compressed: bool + compressed: bool + encoding: str # bech32, what other options? + double_spend: bool + script: str # can be an empty string + redeemscript: str + sequence: int + signatures: List[str] + sigs_required: int + # this is probably an optional but I don't know if it is a string or an int + locktime_cltv: Optional[str | int] + locktime_csv: Optional[str | int] + public_hash: str + script_code: str + unlocking_script: str + unlocking_script_unsigned: str + witness_type: str # segwit, what are other options + witness: Optional[str] + sort: bool + valid: Optional[bool] + + +class OutputDto(BaseModel): + value: int # sats + script: str + script_type: str # p2wpkh, what other options? + public_key: str + public_hash: str + address: str + output_n: int + spent: bool + spending_txid: str + spending_index_n: Optional[int] + + +class TransactionDetailDto(BaseModel): + txid: str + date: Optional[str] + network: str # bitcoin, what are the other options? + witness_type: str # segwit, what are the other options? + coinbase: bool + flag: int + txhash: str + confirmations: Optional[int] + block_height: Optional[int] + block_hash: Optional[str] + fee: Optional[int] + fee_per_kb: Optional[int] + inputs: List[InputDto] + outputs: List[OutputDto] + input_total: int + output_total: int + version: int + locktime: int + raw: str + size: int + vsize: int + verified: bool + status: str # new, not sure the other types + + class GetUtxosRequestDto(BaseModel): fee_rate: str = Field(default="1") transactions: list[TransactionDto] @@ -45,3 +114,7 @@ class UtxoData(BaseModel): class GetAllUtxosResponseDto(BaseModel): utxos: list[UtxoData] + + +class GetAllTransactionsResponseDto(BaseModel): + transactions: list[TransactionDetailDto] diff --git a/backend/src/services/wallet/wallet.py b/backend/src/services/wallet/wallet.py index a7b3255f..481cd516 100644 --- a/backend/src/services/wallet/wallet.py +++ b/backend/src/services/wallet/wallet.py @@ -1,10 +1,14 @@ -import json -import socket from dataclasses import dataclass import bdkpython as bdk +from bitcoinlib.transactions import Transaction from typing import Any, Literal, Optional, cast, List +from src.api import electrum_request, parse_electrum_url - +from src.api.electrum import ( + ElectrumMethod, + GetTransactionsRequestParams, + GetTransactionsResponse, +) from src.database import DB from src.models.wallet import Wallet from src.my_types import ( @@ -22,7 +26,6 @@ from dependency_injector.wiring import inject import structlog -from bitcoinlib.transactions import Transaction LOGGER = structlog.get_logger() @@ -41,56 +44,6 @@ class GetFeeEstimateForUtxoResponseType: psbt: Optional[bdk.PartiallySignedTransaction] -def electrum_request( - url: str, - port: int, - method: str, - # TODO improve typing here - params: None | List[str | bool], - request_id: Optional[int] = 1, -): - if params is None: - params = [] - # Create a TCP socket - try: - with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: - s.connect((url, port)) - - request = json.dumps( - {"jsonrpc": "2.0", "id": request_id, "method": method, "params": params} - ) - - # Send the request - s.sendall((request + "\n").encode("utf-8")) - - # Receive the response - response = b"" - while True: - part = s.recv(4096) - response += part - if len(part) < 4096: - break - - # Decode and parse the JSON response - response_data = json.loads(response.decode("utf-8").strip()) - return response_data - # TODO improve logging and error handling - except socket.error as e: - print(f"Socket error: {e}") - return {"error": str(e)} - except json.JSONDecodeError as e: - print(f"JSON decode error: {e}") - return {"error": "Invalid JSON response"} - except Exception as e: - print(f"An error occurred: {e}") - return {"error": str(e)} - - -def parse_electrum_url(electrum_url: str) -> tuple[Optional[str], Optional[str]]: - url, port = electrum_url.split(":") - return url, port - - class WalletService: """Initiate a wallet using the bdk library and offer various methods to interact with it. The wallet will use an electrum server to obtain data around it's transaction history and current utxos. @@ -310,46 +263,45 @@ def get_all_utxos(self) -> List[bdk.LocalUtxo]: utxos = self.wallet.list_unspent() return utxos - # TODO add return typing def get_all_transactions( self, - ) -> List[Any]: + ) -> List[Transaction]: """Get all transactions for the current wallet.""" wallet_details = Wallet.get_current_wallet() if self.wallet is None or wallet_details is None: - # TODO should I throw an error here? + LOGGER.error("No electrum wallet or wallet details found.") return [] electrum_url = wallet_details.electrum_url if electrum_url is None: - # TODO should I throw an error here? + LOGGER.error("No electrum url found in the wallet details") return [] url, port = parse_electrum_url(electrum_url) if url is None or port is None: - # TODO should I throw an error here? + LOGGER.error("No electrum url or port found in the wallet details") return [] transactions = self.wallet.list_transactions(False) - all_tx_details = [] + all_tx_details: List[Transaction] = [] for index, transaction in enumerate(transactions): - result = electrum_request( + electrum_response = electrum_request( url, int(port), - "blockchain.transaction.get", - [transaction.txid, False], + ElectrumMethod.GET_TRANSACTIONS, + GetTransactionsRequestParams(transaction.txid, False), index, ) - # Decode the transaction - transaction = Transaction.parse(result["result"], strict=True) - # TODO add logging? - - # TODO return as a dataclass? - all_tx_details.append(transaction.as_dict()) + if ( + electrum_response.status == "success" + and electrum_response.data is not None + ): + transaction: Transaction = electrum_response.data + all_tx_details.append(electrum_response.data) return all_tx_details def get_utxos_info(self, utxos_wanted: List[bdk.OutPoint]) -> List[bdk.LocalUtxo]: diff --git a/src/app/api/api.ts b/src/app/api/api.ts index 17877c0a..1cf44fb5 100644 --- a/src/app/api/api.ts +++ b/src/app/api/api.ts @@ -15,6 +15,7 @@ import { HardwareWalletSetPassphraseResponseType, HardwareWalletCloseAndRemoveResponseType, GetBTCPriceResponseType, + GetTransactionsResponseType, } from './types'; import { Network } from '../types/network'; @@ -57,13 +58,12 @@ export class ApiClient { static async getTransactions() { const response = - await fetchHandler(`${configs.backendServerBaseUrl}/utxos/transactions + await fetchHandler(`${configs.backendServerBaseUrl}/transactions `); const data = await response.json(); - // TODO add a type - return data; //as GetUtxosResponseType; + return data as GetTransactionsResponseType; } static async createTxFeeEstimation( utxos: UtxoRequestParam[], diff --git a/src/app/api/types.ts b/src/app/api/types.ts index 74ec4d1b..4f8b57e6 100644 --- a/src/app/api/types.ts +++ b/src/app/api/types.ts @@ -103,3 +103,70 @@ export type GetBTCPriceResponseType = { AUD: number; JPY: number; }; + +type TransactionInputType = { + index_n: number; + prev_txid: string; + output_n: number; + script_type: string; // e.g., "sig_pubkey" + address: string; + value: number; + public_keys: string; + compressed: boolean; + encoding: string; // e.g., "bech32" + double_spend: boolean; + script: string; // can be an empty string + redeemscript: string; + sequence: number; + signatures: string[]; + sigs_required: number; + locktime_cltv?: string | number; + locktime_csv?: string | number; + public_hash: string; + script_code: string; + unlocking_script: string; + unlocking_script_unsigned: string; + witness_type: string; // e.g., "segwit" + witness?: string; + sort: boolean; + valid?: boolean; +}; + +type TransactionOutputType = { + value: number; // in sats + script: string; + script_type: string; // e.g., "p2wpkh" + public_key: string; + public_hash: string; + address: string; + output_n: number; + spent: boolean; + spending_txid: string; + spending_index_n?: number; +}; + +export type GetTransactionsResponseType = { + txid: string; + date?: string; + network: string; // e.g., "bitcoin" + witness_type: string; // e.g., "segwit" + coinbase: boolean; + flag: number; + txhash: string; + confirmations?: number; + block_height?: number; + block_hash?: string; + fee?: number; + fee_per_kb?: number; + inputs: TransactionInputType[]; + outputs: TransactionOutputType[]; + input_total: number; + output_total: number; + version: number; + locktime: number; + raw: string; + size: number; + vsize: number; + verified: boolean; + status: string; // e.g., "new" +}; diff --git a/src/app/hooks/utxos.ts b/src/app/hooks/utxos.ts index 9779c6b2..00a3ea30 100644 --- a/src/app/hooks/utxos.ts +++ b/src/app/hooks/utxos.ts @@ -21,7 +21,7 @@ export function useGetUtxos() { } export function useGetTransactions() { - return useQuery(uxtoQueryKeys.getTransactions, () => ApiClient.getTxos(), { + return useQuery(uxtoQueryKeys.getTransactions, () => ApiClient.getTransactions(), { refetchOnWindowFocus: true, }); } From 15846ef1e2e13bd72bb00a6393ecf3398e39c367 Mon Sep 17 00:00:00 2001 From: Joseph Wyman Date: Sat, 19 Oct 2024 12:49:56 -0400 Subject: [PATCH 06/85] implement getting all outputs that belong to the users wallet --- backend/src/controllers/transactions.py | 24 ++++++++++++++--- backend/src/my_types/__init__.py | 1 + .../my_types/controller_types/utxos_dtos.py | 4 +++ backend/src/services/wallet/wallet.py | 27 ++++++++++++++----- src/app/api/api.ts | 13 ++++++++- src/app/api/types.ts | 4 +++ src/app/hooks/utxos.ts | 13 ++++++++- src/app/pages/Privacy.tsx | 4 ++- 8 files changed, 78 insertions(+), 12 deletions(-) diff --git a/backend/src/controllers/transactions.py b/backend/src/controllers/transactions.py index 71b81b63..5d478f5e 100644 --- a/backend/src/controllers/transactions.py +++ b/backend/src/controllers/transactions.py @@ -5,9 +5,7 @@ from src.containers.service_container import ServiceContainer import structlog -from src.my_types import ( - GetAllTransactionsResponseDto, -) +from src.my_types import GetAllTransactionsResponseDto, GetAllOutputsResponseDto from src.my_types.controller_types.generic_response_types import SimpleErrorResponse transactions_page = Blueprint("get_transactions", __name__, url_prefix="/transactions") @@ -33,3 +31,23 @@ def get_txos( except Exception as e: LOGGER.error("error getting txos", error=e) return SimpleErrorResponse(message="error getting txos").model_dump() + + +@transactions_page.route("/outputs") +@inject +def get_outputs( + wallet_service: WalletService = Provide[ServiceContainer.wallet_service], +): + """ + Get all past and current outputs from the wallet. + """ + try: + outputs = wallet_service.get_all_outputs() + + return GetAllOutputsResponseDto.model_validate( + dict(outputs=[output.as_dict() for output in outputs]) + ).model_dump() + + except Exception as e: + LOGGER.error("error getting outputs", error=e) + return SimpleErrorResponse(message="error getting txos").model_dump() diff --git a/backend/src/my_types/__init__.py b/backend/src/my_types/__init__.py index 8e102267..f0311cdd 100644 --- a/backend/src/my_types/__init__.py +++ b/backend/src/my_types/__init__.py @@ -7,6 +7,7 @@ GetUtxosErrorResponseDto, GetAllUtxosResponseDto, GetAllTransactionsResponseDto, + GetAllOutputsResponseDto, ) from src.my_types.controller_types.fees_dtos import GetCurrentFeesResponseDto diff --git a/backend/src/my_types/controller_types/utxos_dtos.py b/backend/src/my_types/controller_types/utxos_dtos.py index a5e64d55..116c518b 100644 --- a/backend/src/my_types/controller_types/utxos_dtos.py +++ b/backend/src/my_types/controller_types/utxos_dtos.py @@ -118,3 +118,7 @@ class GetAllUtxosResponseDto(BaseModel): class GetAllTransactionsResponseDto(BaseModel): transactions: list[TransactionDetailDto] + + +class GetAllOutputsResponseDto(BaseModel): + outputs: list[OutputDto] diff --git a/backend/src/services/wallet/wallet.py b/backend/src/services/wallet/wallet.py index 481cd516..a3d3d6d4 100644 --- a/backend/src/services/wallet/wallet.py +++ b/backend/src/services/wallet/wallet.py @@ -1,13 +1,12 @@ from dataclasses import dataclass import bdkpython as bdk -from bitcoinlib.transactions import Transaction +from bitcoinlib.transactions import Output, Transaction from typing import Any, Literal, Optional, cast, List from src.api import electrum_request, parse_electrum_url from src.api.electrum import ( ElectrumMethod, GetTransactionsRequestParams, - GetTransactionsResponse, ) from src.database import DB from src.models.wallet import Wallet @@ -136,7 +135,8 @@ def connect_wallet( ) wallet_change_descriptor = ( - bdk.Descriptor(change_descriptor, bdk.Network._value2member_map_[network]) + bdk.Descriptor(change_descriptor, + bdk.Network._value2member_map_[network]) if change_descriptor else None ) @@ -158,7 +158,8 @@ def connect_wallet( database_config=db_config, ) - LOGGER.info(f"Connecting a new wallet to electrum server {wallet_details_id}") + LOGGER.info( + f"Connecting a new wallet to electrum server {wallet_details_id}") LOGGER.info(f"xpub {wallet_descriptor.as_string()}") wallet.sync(blockchain, None) @@ -207,7 +208,8 @@ def create_spendable_descriptor( twelve_word_secret = bdk.Mnemonic(bdk.WordCount.WORDS12) # xpriv - descriptor_secret_key = bdk.DescriptorSecretKey(network, twelve_word_secret, "") + descriptor_secret_key = bdk.DescriptorSecretKey( + network, twelve_word_secret, "") wallet_descriptor = None if script_type == ScriptType.P2PKH: @@ -304,6 +306,18 @@ def get_all_transactions( all_tx_details.append(electrum_response.data) return all_tx_details + def get_all_outputs(self) -> List[Output]: + """Get all spent and unspent transaction outputs for the current wallet.""" + all_transactions = self.get_all_transactions() + all_outputs: List[Output] = [] + for transaction in all_transactions: + for output in transaction.outputs: + script = bdk.Script(output.script.raw) + if self.wallet and self.wallet.is_mine(script): + all_outputs.append(output) + + return all_outputs + def get_utxos_info(self, utxos_wanted: List[bdk.OutPoint]) -> List[bdk.LocalUtxo]: """For a given set of txids and the vout pointing to a utxo, return the utxos""" existing_utxos = cast(List[bdk.LocalUtxo], self.get_all_utxos()) @@ -360,7 +374,8 @@ def build_transaction( script, amount_per_recipient_output ) - built_transaction: bdk.TxBuilderResult = tx_builder.finish(self.wallet) + built_transaction: bdk.TxBuilderResult = tx_builder.finish( + self.wallet) built_transaction.transaction_details.transaction return BuildTransactionResponseType( diff --git a/src/app/api/api.ts b/src/app/api/api.ts index 1cf44fb5..3d59a7f1 100644 --- a/src/app/api/api.ts +++ b/src/app/api/api.ts @@ -16,6 +16,7 @@ import { HardwareWalletCloseAndRemoveResponseType, GetBTCPriceResponseType, GetTransactionsResponseType, + GetOutputsResponseType, } from './types'; import { Network } from '../types/network'; @@ -58,13 +59,23 @@ export class ApiClient { static async getTransactions() { const response = - await fetchHandler(`${configs.backendServerBaseUrl}/transactions + await fetchHandler(`${configs.backendServerBaseUrl}/transactions/ `); const data = await response.json(); return data as GetTransactionsResponseType; } + + static async getOutputs() { + const response = + await fetchHandler(`${configs.backendServerBaseUrl}/transactions/outputs +`); + + const data = await response.json(); + + return data as GetOutputsResponseType; + } static async createTxFeeEstimation( utxos: UtxoRequestParam[], feeRate: number = 1, diff --git a/src/app/api/types.ts b/src/app/api/types.ts index 4f8b57e6..0ce0d974 100644 --- a/src/app/api/types.ts +++ b/src/app/api/types.ts @@ -170,3 +170,7 @@ export type GetTransactionsResponseType = { verified: boolean; status: string; // e.g., "new" }; + +export type GetOutputsResponseType = { + outputs: TransactionOutputType[]; +}; diff --git a/src/app/hooks/utxos.ts b/src/app/hooks/utxos.ts index 00a3ea30..9753b578 100644 --- a/src/app/hooks/utxos.ts +++ b/src/app/hooks/utxos.ts @@ -7,6 +7,7 @@ export const uxtoQueryKeys = { getUtxos: ['getUtxos'], getTransactions: ['getTransactions'], getCurrentFees: ['getCurrentFees'], + getOutputs: ['getOutputs'], }; export function useGetBalance() { @@ -21,7 +22,17 @@ export function useGetUtxos() { } export function useGetTransactions() { - return useQuery(uxtoQueryKeys.getTransactions, () => ApiClient.getTransactions(), { + return useQuery( + uxtoQueryKeys.getTransactions, + () => ApiClient.getTransactions(), + { + refetchOnWindowFocus: true, + }, + ); +} + +export function useGetOutputs() { + return useQuery(uxtoQueryKeys.getOutputs, () => ApiClient.getOutputs(), { refetchOnWindowFocus: true, }); } diff --git a/src/app/pages/Privacy.tsx b/src/app/pages/Privacy.tsx index 64c6e4dd..f47cbd54 100644 --- a/src/app/pages/Privacy.tsx +++ b/src/app/pages/Privacy.tsx @@ -1,6 +1,6 @@ import { Tabs, rem } from '@mantine/core'; import { IconCoins, IconArrowsDownUp, IconEye } from '@tabler/icons-react'; -import { useGetTransactions } from '../hooks/utxos'; +import { useGetTransactions, useGetOutputs } from '../hooks/utxos'; export const Privacy = () => { const iconStyle = { width: rem(22), height: rem(22) }; @@ -12,6 +12,8 @@ export const Privacy = () => { const transactionsResponse = useGetTransactions(); console.log('transactionsResponse', transactionsResponse.data); + const outputs = useGetOutputs(); + console.log('outputs', outputs.data); return (
Date: Sun, 20 Oct 2024 09:57:06 -0400 Subject: [PATCH 07/85] add electrum api tests --- backend/src/api/electrum.py | 2 +- backend/src/tests/api_tests/test_electrum.py | 82 ++++++++++++++++++++ 2 files changed, 83 insertions(+), 1 deletion(-) create mode 100644 backend/src/tests/api_tests/test_electrum.py diff --git a/backend/src/api/electrum.py b/backend/src/api/electrum.py index eb6198e1..570609b8 100644 --- a/backend/src/api/electrum.py +++ b/backend/src/api/electrum.py @@ -47,8 +47,8 @@ def electrum_request( params: Optional[ALL_UTXOS_REQUEST_PARAMS], request_id: Optional[int] = 1, ) -> ElectrumResponse: - # Create a TCP socket try: + # Create a IPv4 TCP socket with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: s.connect((url, port)) diff --git a/backend/src/tests/api_tests/test_electrum.py b/backend/src/tests/api_tests/test_electrum.py new file mode 100644 index 00000000..c7d918d0 --- /dev/null +++ b/backend/src/tests/api_tests/test_electrum.py @@ -0,0 +1,82 @@ +from unittest.case import TestCase +import json + +from bitcoinlib.transactions import Transaction +from src.api.electrum import ( + ElectrumResponse, + parse_electrum_url, + electrum_request, + ElectrumMethod, + GetTransactionsRequestParams, +) +from unittest.mock import patch, Mock +import socket + +mock_electrum_get_transactions_response = { + "id": 7, + "jsonrpc": "2.0", + "result": "02000000000101b0d9cd7e4b900ef0ab43c5a215d6a523a4a422e8a19de488cf948bf797ba31260000000000feffffff0249138727010000001600141c96f2fd3abc46a6161500fed55765a32e31a6d32ade7e020000000016001405de346b6fd81b848f5be779e0b59e26283279400247304402206b89fefa04aa1a132b166c098c8b597974492532ac6162b7561d514ddea12b1d02207d9f3444a00429ea9c935ff4f4eb9a31d641e2df1c0cbff3c55be5ed13731b200121028a993097d51522ee09207cbf299fcadfebb6fe7889348aab99d4f25eff40d9b66c000000", +} +mock_electrum_get_transactions_response_json = json.dumps( + mock_electrum_get_transactions_response +) +mock_electrum_get_transactions_response_parsed = Transaction.parse( + mock_electrum_get_transactions_response["result"], strict=True +) + + +class TestElectrumApi(TestCase): + def setUp(self): + self.mock_url = "blockstream" + self.mock_port = 1234 + self.mock_request_id = 432 + self.mock_tx_id = "mockTxId" + + def test_parse_electrum_url(self): + mock_electrum_url = f"{self.mock_url}:{self.mock_port}" + url, port = parse_electrum_url(mock_electrum_url) + assert port == str(self.mock_port) + assert url == self.mock_url + + def test_get_transactions_electrum_request_success(self): + request_method = ElectrumMethod.GET_TRANSACTIONS + + with patch.object(socket, "socket") as mock_socket: + mock_live_socket = Mock() + mock_socket.return_value.__enter__.return_value = mock_live_socket + mock_live_socket.recv.return_value = ( + mock_electrum_get_transactions_response_json.encode("utf-8") + ) + response = electrum_request( + self.mock_url, + self.mock_port, + request_method, + GetTransactionsRequestParams(self.mock_tx_id, False), + self.mock_request_id, + ) + mock_socket.assert_called_with(socket.AF_INET, socket.SOCK_STREAM) + mock_live_socket.connect.assert_called_with((self.mock_url, self.mock_port)) + + mock_live_socket.recv.assert_called_with(4096) + + assert response == ElectrumResponse( + status="success", data=mock_electrum_get_transactions_response_parsed + ) + + def test_get_transactions_electrum_request_error(self): + request_method = ElectrumMethod.GET_TRANSACTIONS + + with patch.object(socket, "socket") as mock_socket: + mock_live_socket = Mock() + mock_socket.return_value.__enter__.return_value = mock_live_socket + # bad electrum response + mock_live_socket.recv.return_value = None + response = electrum_request( + self.mock_url, + self.mock_port, + request_method, + GetTransactionsRequestParams(self.mock_tx_id, False), + self.mock_request_id, + ) + + assert response == ElectrumResponse(status="error", data=None) From 159b8230e2815b8eb726876ad25354c521f68116 Mon Sep 17 00:00:00 2001 From: Joseph Wyman Date: Sun, 20 Oct 2024 10:39:04 -0400 Subject: [PATCH 08/85] add controller test for getting all transactions --- backend/src/tests/api_tests/test_electrum.py | 18 +++-------- .../test_transactions_controller.py | 31 +++++++++++++++++++ backend/src/tests/mocks.py | 20 ++++++++++++ 3 files changed, 55 insertions(+), 14 deletions(-) create mode 100644 backend/src/tests/controller_tests/test_transactions_controller.py diff --git a/backend/src/tests/api_tests/test_electrum.py b/backend/src/tests/api_tests/test_electrum.py index c7d918d0..bb21fb22 100644 --- a/backend/src/tests/api_tests/test_electrum.py +++ b/backend/src/tests/api_tests/test_electrum.py @@ -1,7 +1,5 @@ from unittest.case import TestCase -import json -from bitcoinlib.transactions import Transaction from src.api.electrum import ( ElectrumResponse, parse_electrum_url, @@ -10,19 +8,11 @@ GetTransactionsRequestParams, ) from unittest.mock import patch, Mock -import socket - -mock_electrum_get_transactions_response = { - "id": 7, - "jsonrpc": "2.0", - "result": "02000000000101b0d9cd7e4b900ef0ab43c5a215d6a523a4a422e8a19de488cf948bf797ba31260000000000feffffff0249138727010000001600141c96f2fd3abc46a6161500fed55765a32e31a6d32ade7e020000000016001405de346b6fd81b848f5be779e0b59e26283279400247304402206b89fefa04aa1a132b166c098c8b597974492532ac6162b7561d514ddea12b1d02207d9f3444a00429ea9c935ff4f4eb9a31d641e2df1c0cbff3c55be5ed13731b200121028a993097d51522ee09207cbf299fcadfebb6fe7889348aab99d4f25eff40d9b66c000000", -} -mock_electrum_get_transactions_response_json = json.dumps( - mock_electrum_get_transactions_response -) -mock_electrum_get_transactions_response_parsed = Transaction.parse( - mock_electrum_get_transactions_response["result"], strict=True +from src.tests.mocks import ( + mock_electrum_get_transactions_response_json, + mock_electrum_get_transactions_response_parsed, ) +import socket class TestElectrumApi(TestCase): diff --git a/backend/src/tests/controller_tests/test_transactions_controller.py b/backend/src/tests/controller_tests/test_transactions_controller.py new file mode 100644 index 00000000..709d08d2 --- /dev/null +++ b/backend/src/tests/controller_tests/test_transactions_controller.py @@ -0,0 +1,31 @@ +from unittest import TestCase + +from unittest.mock import MagicMock +from src.app import AppCreator +from src.services.wallet.wallet import WalletService +from src.tests.mocks import all_transactions_mock +import json + + +class TestTransactionsController(TestCase): + def setUp(self): + app_creator = AppCreator() + self.app = app_creator.create_app() + self.test_client = self.app.test_client() + self.mock_wallet_service = MagicMock(WalletService) + self.mock_wallet_class = MagicMock( + WalletService, return_value=self.mock_wallet_service + ) + + def test_get_transactions(self): + get_all_transactions_mock = MagicMock(return_value=all_transactions_mock) + with self.app.container.wallet_service.override(self.mock_wallet_service): + self.mock_wallet_service.get_all_transactions = get_all_transactions_mock + get_transactions_response = self.test_client.get("/transactions/") + + get_all_transactions_mock.assert_called_once() + + assert get_transactions_response.status == "200 OK" + assert json.loads(get_transactions_response.data) == { + "transactions": [tx.as_dict() for tx in all_transactions_mock] + } diff --git a/backend/src/tests/mocks.py b/backend/src/tests/mocks.py index b6ab65fb..ed1d5c14 100644 --- a/backend/src/tests/mocks.py +++ b/backend/src/tests/mocks.py @@ -1,6 +1,8 @@ +from bitcoinlib.transactions import Transaction from src.my_types import ( FeeDetails, ) +import json import bdkpython as bdk tx_out_mock = bdk.TxOut(value=1000, script_pubkey="mock_script_pubkey") @@ -22,3 +24,21 @@ fee_details_mock = FeeDetails(percent_fee_is_of_utxo=0.1, fee=100) + + +mock_electrum_get_transactions_response = { + "id": 7, + "jsonrpc": "2.0", + "result": "02000000000101b0d9cd7e4b900ef0ab43c5a215d6a523a4a422e8a19de488cf948bf797ba31260000000000feffffff0249138727010000001600141c96f2fd3abc46a6161500fed55765a32e31a6d32ade7e020000000016001405de346b6fd81b848f5be779e0b59e26283279400247304402206b89fefa04aa1a132b166c098c8b597974492532ac6162b7561d514ddea12b1d02207d9f3444a00429ea9c935ff4f4eb9a31d641e2df1c0cbff3c55be5ed13731b200121028a993097d51522ee09207cbf299fcadfebb6fe7889348aab99d4f25eff40d9b66c000000", +} +mock_electrum_get_transactions_response_json = json.dumps( + mock_electrum_get_transactions_response +) +mock_electrum_get_transactions_response_parsed = Transaction.parse( + mock_electrum_get_transactions_response["result"], strict=True +) + + +all_transactions_mock = [ + Transaction.parse(mock_electrum_get_transactions_response["result"], strict=True) +] From 052d0021cc96f2253b1143b94f79f34a127479c2 Mon Sep 17 00:00:00 2001 From: Joseph Wyman Date: Sun, 20 Oct 2024 11:55:30 -0400 Subject: [PATCH 09/85] add test for get all outputs controller --- .../test_transactions_controller.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/backend/src/tests/controller_tests/test_transactions_controller.py b/backend/src/tests/controller_tests/test_transactions_controller.py index 709d08d2..52a8d90d 100644 --- a/backend/src/tests/controller_tests/test_transactions_controller.py +++ b/backend/src/tests/controller_tests/test_transactions_controller.py @@ -1,10 +1,12 @@ from unittest import TestCase from unittest.mock import MagicMock +from typing import List from src.app import AppCreator from src.services.wallet.wallet import WalletService from src.tests.mocks import all_transactions_mock import json +from bitcoinlib.transactions import Output class TestTransactionsController(TestCase): @@ -29,3 +31,18 @@ def test_get_transactions(self): assert json.loads(get_transactions_response.data) == { "transactions": [tx.as_dict() for tx in all_transactions_mock] } + + def test_get_utxos(self): + output_lists: List[List[Output]] = [tx.outputs for tx in all_transactions_mock] + all_outputs = [output for output_list in output_lists for output in output_list] + get_all_outputs_mock = MagicMock(return_value=all_outputs) + with self.app.container.wallet_service.override(self.mock_wallet_service): + self.mock_wallet_service.get_all_outputs = get_all_outputs_mock + get_all_outputs_response = self.test_client.get("transactions/outputs") + + get_all_outputs_mock.assert_called_once() + + assert get_all_outputs_response.status == "200 OK" + assert json.loads(get_all_outputs_response.data) == { + "outputs": [output.as_dict() for output in all_outputs] + } From c33ded59db6693503e40b2a9f1d30c64f8c59760 Mon Sep 17 00:00:00 2001 From: Joseph Wyman Date: Sun, 20 Oct 2024 13:34:38 -0400 Subject: [PATCH 10/85] add test for get all transaction service method --- backend/src/api/electrum.py | 9 +- backend/src/services/wallet/wallet.py | 12 +- backend/src/tests/api_tests/test_electrum.py | 6 + .../service_tests/test_wallet_service.py | 116 +++++++++++++++++- 4 files changed, 132 insertions(+), 11 deletions(-) diff --git a/backend/src/api/electrum.py b/backend/src/api/electrum.py index 570609b8..45da3ac4 100644 --- a/backend/src/api/electrum.py +++ b/backend/src/api/electrum.py @@ -10,8 +10,13 @@ def parse_electrum_url(electrum_url: str) -> tuple[Optional[str], Optional[str]]: - url, port = electrum_url.split(":") - return url, port + try: + url, port = electrum_url.split(":") + if url == "" or port == "": + return None, None + return url, port + except ValueError: + return None, None class ElectrumMethod(Enum): diff --git a/backend/src/services/wallet/wallet.py b/backend/src/services/wallet/wallet.py index a3d3d6d4..321fdb6c 100644 --- a/backend/src/services/wallet/wallet.py +++ b/backend/src/services/wallet/wallet.py @@ -135,8 +135,7 @@ def connect_wallet( ) wallet_change_descriptor = ( - bdk.Descriptor(change_descriptor, - bdk.Network._value2member_map_[network]) + bdk.Descriptor(change_descriptor, bdk.Network._value2member_map_[network]) if change_descriptor else None ) @@ -158,8 +157,7 @@ def connect_wallet( database_config=db_config, ) - LOGGER.info( - f"Connecting a new wallet to electrum server {wallet_details_id}") + LOGGER.info(f"Connecting a new wallet to electrum server {wallet_details_id}") LOGGER.info(f"xpub {wallet_descriptor.as_string()}") wallet.sync(blockchain, None) @@ -208,8 +206,7 @@ def create_spendable_descriptor( twelve_word_secret = bdk.Mnemonic(bdk.WordCount.WORDS12) # xpriv - descriptor_secret_key = bdk.DescriptorSecretKey( - network, twelve_word_secret, "") + descriptor_secret_key = bdk.DescriptorSecretKey(network, twelve_word_secret, "") wallet_descriptor = None if script_type == ScriptType.P2PKH: @@ -374,8 +371,7 @@ def build_transaction( script, amount_per_recipient_output ) - built_transaction: bdk.TxBuilderResult = tx_builder.finish( - self.wallet) + built_transaction: bdk.TxBuilderResult = tx_builder.finish(self.wallet) built_transaction.transaction_details.transaction return BuildTransactionResponseType( diff --git a/backend/src/tests/api_tests/test_electrum.py b/backend/src/tests/api_tests/test_electrum.py index bb21fb22..773a38b6 100644 --- a/backend/src/tests/api_tests/test_electrum.py +++ b/backend/src/tests/api_tests/test_electrum.py @@ -28,6 +28,12 @@ def test_parse_electrum_url(self): assert port == str(self.mock_port) assert url == self.mock_url + def test_parse_electrum_url_without_port(self): + mock_electrum_url = f"{self.mock_url}:" + url, port = parse_electrum_url(mock_electrum_url) + assert port is None + assert url is None + def test_get_transactions_electrum_request_success(self): request_method = ElectrumMethod.GET_TRANSACTIONS diff --git a/backend/src/tests/service_tests/test_wallet_service.py b/backend/src/tests/service_tests/test_wallet_service.py index 1f0a7416..cc187fc2 100644 --- a/backend/src/tests/service_tests/test_wallet_service.py +++ b/backend/src/tests/service_tests/test_wallet_service.py @@ -2,6 +2,12 @@ import os from unittest.mock import MagicMock, call, patch, Mock import pytest +from src.api.electrum import ( + ElectrumMethod, + ElectrumResponse, + GetTransactionsRequestParams, + GetTransactionsResponse, +) from src.models.wallet import Wallet from src.services import WalletService from src.services.wallet.wallet import ( @@ -14,7 +20,11 @@ ) from src.my_types import GetUtxosRequestDto from src.my_types.script_types import ScriptType -from src.tests.mocks import local_utxo_mock, transaction_details_mock +from src.tests.mocks import ( + local_utxo_mock, + transaction_details_mock, + all_transactions_mock, +) from typing import cast @@ -859,3 +869,107 @@ def test_create_spendable_wallet(self): assert response == wallet_mock wallet_sync_mock.assert_called_with(block_chain_mock, None) + + def test_get_all_transactions_success(self): + with ( + patch("src.services.wallet.wallet.Wallet") as wallet_model_patch, + patch( + "src.services.wallet.wallet.electrum_request" + ) as mock_electrum_request, + ): + mock_wallet = MagicMock() + # mock the transactions that bdk fetches from the wallet + mock_wallet.list_transactions.return_value = [ + Mock(txid="txid1"), + Mock(txid="txid2"), + ] + + wallet_details_mock = MagicMock() + wallet_details_mock.electrum_url = "blockstream:1234" + + all_transactions_from_electrum = [ + all_transactions_mock[0], + all_transactions_mock[0], + ] + all_transactions_from_electrum[1].txid = "txid2" + + mock_electrum_request.side_effect = [ + ElectrumResponse( + status="success", data=all_transactions_from_electrum[0] + ), + ElectrumResponse( + status="success", data=all_transactions_from_electrum[1] + ), + ] + wallet_model_patch.get_current_wallet.return_value = wallet_details_mock + self.wallet_service.wallet = mock_wallet + + # call the method we are testing + get_all_transactions_response = self.wallet_service.get_all_transactions() + + mock_wallet.list_transactions.assert_called_with(False) + + assert mock_electrum_request.call_count == 2 + mock_electrum_request.assert_any_call( + "blockstream", + 1234, + ElectrumMethod.GET_TRANSACTIONS, + GetTransactionsRequestParams("txid1", False), + 0, + ) + + mock_electrum_request.assert_any_call( + "blockstream", + 1234, + ElectrumMethod.GET_TRANSACTIONS, + GetTransactionsRequestParams("txid2", False), + 1, + ) + + assert get_all_transactions_response == all_transactions_from_electrum + + def test_get_all_transactions_without_url(self): + with ( + patch("src.services.wallet.wallet.Wallet") as wallet_model_patch, + patch( + "src.services.wallet.wallet.electrum_request" + ) as mock_electrum_request, + ): + mock_wallet = MagicMock() + mock_wallet.list_transactions = MagicMock() + wallet_details_mock = MagicMock() + # set no electrum url + wallet_details_mock.electrum_url = None + + wallet_model_patch.get_current_wallet.return_value = wallet_details_mock + self.wallet_service.wallet = mock_wallet + + # call the method we are testing + get_all_transactions_response = self.wallet_service.get_all_transactions() + + mock_wallet.list_transactions.assert_not_called() + assert mock_electrum_request.call_count == 0 + assert get_all_transactions_response == [] + + def test_get_all_transactions_without_port(self): + with ( + patch("src.services.wallet.wallet.Wallet") as wallet_model_patch, + patch( + "src.services.wallet.wallet.electrum_request" + ) as mock_electrum_request, + ): + mock_wallet = MagicMock() + mock_wallet.list_transactions = MagicMock() + wallet_details_mock = MagicMock() + # set no electrum port + wallet_details_mock.electrum_url = "blockstreamwithoutPort" + + wallet_model_patch.get_current_wallet.return_value = wallet_details_mock + self.wallet_service.wallet = mock_wallet + + # call the method we are testing + get_all_transactions_response = self.wallet_service.get_all_transactions() + + mock_wallet.list_transactions.assert_not_called() + assert mock_electrum_request.call_count == 0 + assert get_all_transactions_response == [] From 3d829a13c1873f7803198f91cf6caa62d970ecaf Mon Sep 17 00:00:00 2001 From: Joseph Wyman Date: Sun, 20 Oct 2024 15:11:02 -0400 Subject: [PATCH 11/85] add test for get_all_outputs --- .../service_tests/test_wallet_service.py | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/backend/src/tests/service_tests/test_wallet_service.py b/backend/src/tests/service_tests/test_wallet_service.py index cc187fc2..ecf4bc32 100644 --- a/backend/src/tests/service_tests/test_wallet_service.py +++ b/backend/src/tests/service_tests/test_wallet_service.py @@ -973,3 +973,43 @@ def test_get_all_transactions_without_port(self): mock_wallet.list_transactions.assert_not_called() assert mock_electrum_request.call_count == 0 assert get_all_transactions_response == [] + + def test_get_all_outputs(self): + mock_wallet = Mock() + self.wallet_service.wallet = mock_wallet + mock_wallet.is_mine = Mock() + # mark first output as mine and the second as not + mock_wallet.is_mine.side_effect = [True, False] + self.wallet_service.get_all_transactions = Mock( + return_value=all_transactions_mock + ) + + # call the method we are testing + get_all_outputs_response = self.wallet_service.get_all_outputs() + + self.wallet_service.get_all_transactions.assert_called() + + assert mock_wallet.is_mine.call_count == 2 + + # the returned outputs should only be the first one since we mocked out that the second output would return False when checking if it is mine + assert get_all_outputs_response == [all_transactions_mock[0].outputs[0]] + + def test_get_all_outputs_if_none_are_mine(self): + mock_wallet = Mock() + self.wallet_service.wallet = mock_wallet + mock_wallet.is_mine = Mock() + # mark all as False + mock_wallet.is_mine.return_value = False + self.wallet_service.get_all_transactions = Mock( + return_value=all_transactions_mock + ) + + # call the method we are testing + get_all_outputs_response = self.wallet_service.get_all_outputs() + + self.wallet_service.get_all_transactions.assert_called() + + assert mock_wallet.is_mine.call_count == 2 + + # no outputs are mine so an empty list should be returned + assert get_all_outputs_response == [] From 14e8d7b43d23e67b58938e1de56108ddca3d0729 Mon Sep 17 00:00:00 2001 From: Joseph Wyman Date: Sun, 20 Oct 2024 16:36:56 -0400 Subject: [PATCH 12/85] add initial txo table look --- package-lock.json | 9 ++ package.json | 1 + src/app/components/privacy/txosTable.tsx | 105 +++++++++++++++-------- src/app/pages/Home.tsx | 2 +- src/app/pages/Privacy.tsx | 14 ++- 5 files changed, 92 insertions(+), 39 deletions(-) diff --git a/package-lock.json b/package-lock.json index 991a01eb..e5380b07 100644 --- a/package-lock.json +++ b/package-lock.json @@ -26,6 +26,7 @@ "path-browserify": "^1.0.1", "react": "^18.2.0", "react-dom": "^18.2.0", + "react-icons": "^5.3.0", "react-query": "^3.39.3", "react-router-dom": "^6.16.0", "tree-kill": "^1.2.2" @@ -16937,6 +16938,14 @@ "react": "^18.2.0" } }, + "node_modules/react-icons": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/react-icons/-/react-icons-5.3.0.tgz", + "integrity": "sha512-DnUk8aFbTyQPSkCfF8dbX6kQjXA9DktMeJqfjrg6cK9vwQVMxmcA3BfP4QoiztVmEHtwlTgLFsPuH2NskKT6eg==", + "peerDependencies": { + "react": "*" + } + }, "node_modules/react-is": { "version": "17.0.2", "resolved": "https://registry.npmjs.org/react-is/-/react-is-17.0.2.tgz", diff --git a/package.json b/package.json index ad4bcccb..e1fc8c49 100644 --- a/package.json +++ b/package.json @@ -111,6 +111,7 @@ "path-browserify": "^1.0.1", "react": "^18.2.0", "react-dom": "^18.2.0", + "react-icons": "^5.3.0", "react-query": "^3.39.3", "react-router-dom": "^6.16.0", "tree-kill": "^1.2.2" diff --git a/src/app/components/privacy/txosTable.tsx b/src/app/components/privacy/txosTable.tsx index f5298603..89abdfbd 100644 --- a/src/app/components/privacy/txosTable.tsx +++ b/src/app/components/privacy/txosTable.tsx @@ -6,24 +6,42 @@ import { useMaterialReactTable, } from 'material-react-table'; -import { CopyButton, rem, Tooltip, ActionIcon } from '@mantine/core'; -import { IconCheck, IconCopy } from '@tabler/icons-react'; +import { + CopyButton, + rem, + Tooltip, + ActionIcon, + Chip, + ChipGroup, +} from '@mantine/core'; +import { MdLabelOutline } from 'react-icons/md'; + +import { + IconCheck, + IconCircleCheck, + IconCircleX, + IconCopy, +} from '@tabler/icons-react'; +import { BtcMetric, btcSatHandler } from '../../types/btcSatHandler'; const sectionColor = 'rgb(1, 67, 97)'; -type TxosTableProps = {}; +type TxosTableProps = { + txos: any; + btcMetric: BtcMetric; +}; -export const TxosTable = ({}: TxosTableProps) => { +export const TxosTable = ({ txos, btcMetric }: TxosTableProps) => { const columns = useMemo(() => { const defaultColumns = [ { - header: 'Txid', + header: 'Address', size: 100, - accessorKey: 'txid', + accessorKey: 'address', Cell: ({ row }: { row: any }) => { - const prefix = row.original.txid.substring(0, 4); - const suffix = row.original.txid.substring( - row.original.txid.length - 4, + const prefix = row.original.address.substring(0, 4); + const suffix = row.original.address.substring( + row.original?.address?.length - 4, ); const abrv = `${prefix}....${suffix}`; return ( @@ -61,45 +79,61 @@ export const TxosTable = ({}: TxosTableProps) => { accessorKey: 'amount', size: 100, Cell: ({ row }: { row: any }) => { - return ( -
-

amount

-
+ const amount = btcSatHandler( + Number(row.original.value).toFixed(2).toLocaleString(), + btcMetric, ); - }, - }, - { - header: '$ Amount', - accessorKey: 'amountUSD', - size: 100, - Cell: ({ row }: { row: any }) => { return (
-

{'amountUSDDisplay'}

+

+ {btcMetric === BtcMetric.BTC + ? amount + : Number(amount).toLocaleString()} +

); }, }, { - header: '~ Fee %', - accessorKey: 'selfCost', - size: 100, - Cell: ({ row }: { row: any }) => { + header: 'Labels', + accessorKey: 'label', + size: 250, + Cell: () => { + // mock labels for now + const allLabels = ['do not spend', 'bad change', "another"]; return ( -
-

{'hi'}

+
+ {allLabels.map((label) => ( + } + color="red" + checked={true} + variant="light" + className="mr-1" + classNames={{ + checkIcon: 'h-0 hidden', + }} + > + {label} + + ))}
); }, }, { - header: '$ Fee', - accessorKey: 'feeUSD', + header: 'Unspent', + accessorKey: 'spent', size: 100, - Cell: () => { + Cell: ({ row }) => { + const isSpent = row.original.spent === true; return ( -
-

{'amountUSDDisplay'}

+
+ {isSpent ? ( + + ) : ( + + )}
); }, @@ -107,11 +141,11 @@ export const TxosTable = ({}: TxosTableProps) => { ]; return defaultColumns; - }, []); + }, [txos, btcMetric]); const table = useMaterialReactTable({ columns, - data: [], + data: txos, enableRowSelection: false, enableDensityToggle: false, enableFullScreenToggle: false, @@ -139,8 +173,7 @@ export const TxosTable = ({}: TxosTableProps) => { }} className="text-2xl font-semibold" > - {'mock title'} - (utxos) + Outputs

); diff --git a/src/app/pages/Home.tsx b/src/app/pages/Home.tsx index 2e5dbc2a..364f81cd 100644 --- a/src/app/pages/Home.tsx +++ b/src/app/pages/Home.tsx @@ -497,7 +497,7 @@ function Home() {
{displayType === DisplayType.PRIVACY ? ( - + ) : (
{ +type PrivacyProps = { + btcMetric: BtcMetric; +}; +export const Privacy = ({ btcMetric }: PrivacyProps) => { const iconStyle = { width: rem(22), height: rem(22) }; enum PrivacyTabs { UTXOS_STXOS = 'utxos_stxos', @@ -46,7 +51,12 @@ export const Privacy = () => { - My utxos and stxos +
+ +
From c3ec21b786af392cbed72202aa70d042dd9bc385 Mon Sep 17 00:00:00 2001 From: Joseph Wyman Date: Tue, 22 Oct 2024 19:57:45 -0400 Subject: [PATCH 13/85] add initial label adding and removing to FE and BE --- backend/src/app.py | 3 +- backend/src/controllers/transactions.py | 115 +++++++++++++++++- backend/src/database.py | 24 ++++ backend/src/models/label.py | 33 +++++ backend/src/models/output_labels.py | 25 ++++ backend/src/models/outputs.py | 23 ++++ backend/src/my_types/__init__.py | 5 + .../my_types/controller_types/utxos_dtos.py | 38 +++++- backend/src/my_types/transactions.py | 30 +++++ backend/src/services/wallet/wallet.py | 103 +++++++++++++++- src/app/api/api.ts | 42 +++++++ src/app/api/types.ts | 36 +++++- src/app/components/OutputModal.tsx | 95 +++++++++++++++ src/app/components/privacy/txosTable.tsx | 98 ++++++++++----- src/app/hooks/transactions.ts | 63 ++++++++++ src/app/hooks/utxos.ts | 16 --- src/app/pages/Privacy.tsx | 2 +- 17 files changed, 694 insertions(+), 57 deletions(-) create mode 100644 backend/src/models/label.py create mode 100644 backend/src/models/output_labels.py create mode 100644 backend/src/models/outputs.py create mode 100644 backend/src/my_types/transactions.py create mode 100644 src/app/components/OutputModal.tsx create mode 100644 src/app/hooks/transactions.ts diff --git a/backend/src/app.py b/backend/src/app.py index a87e215d..6bb6bb42 100644 --- a/backend/src/app.py +++ b/backend/src/app.py @@ -1,6 +1,6 @@ from flask import Flask, request from flask_cors import CORS -from src.database import DB +from src.database import DB, populate_labels # initialize structlog from src.utils import logging # noqa: F401, E261 @@ -94,6 +94,7 @@ def setup_database(app): DB.init_app(app) with app.app_context(): DB.create_all() + populate_labels() # for some reason the frontend doesn't run the executable with app.y being __main__ diff --git a/backend/src/controllers/transactions.py b/backend/src/controllers/transactions.py index 5d478f5e..e1e9a6a5 100644 --- a/backend/src/controllers/transactions.py +++ b/backend/src/controllers/transactions.py @@ -1,11 +1,20 @@ -from flask import Blueprint +from flask import Blueprint, request +import json from src.services import WalletService from dependency_injector.wiring import inject, Provide from src.containers.service_container import ServiceContainer import structlog -from src.my_types import GetAllTransactionsResponseDto, GetAllOutputsResponseDto +from src.my_types import ( + GetAllTransactionsResponseDto, + GetAllOutputsResponseDto, + AddOutputLabelRequestDto, + RemoveOutputLabelRequestDto, + RemoveOutputLabelResponseDto, + AddOutputLabelResponseDto, + GetOutputLabelsResponseDto, +) from src.my_types.controller_types.generic_response_types import SimpleErrorResponse transactions_page = Blueprint("get_transactions", __name__, url_prefix="/transactions") @@ -51,3 +60,105 @@ def get_outputs( except Exception as e: LOGGER.error("error getting outputs", error=e) return SimpleErrorResponse(message="error getting txos").model_dump() + + +@transactions_page.route("/outputs/labels", methods=["GET"]) +@inject +def get_output_labels( + wallet_service: WalletService = Provide[ServiceContainer.wallet_service], +): + """ + Get all output labels. + """ + try: + labels = wallet_service.get_output_labels() + + return GetOutputLabelsResponseDto.model_validate( + dict(labels=labels) + ).model_dump() + # + except Exception as e: + LOGGER.error("error getting all output labels", error=e) + return SimpleErrorResponse( + message="error getting all output labels" + ).model_dump() + + +@transactions_page.route("/outputs/label", methods=["POST"]) +@inject +def add_output_label( + wallet_service: WalletService = Provide[ServiceContainer.wallet_service], +): + """ + Add a label to a specific output (defined by the txid and vout). + """ + try: + add_output_label_request_dto = AddOutputLabelRequestDto.model_validate( + json.loads(request.data) + ) + labels = wallet_service.add_label_to_output( + add_output_label_request_dto.txid, + add_output_label_request_dto.vout, + add_output_label_request_dto.labelName, + ) + + return AddOutputLabelResponseDto.model_validate( + dict( + labels=[ + dict( + label=label.name, + display_name=label.display_name, + description=label.description, + ) + for label in labels + ] + ) + ).model_dump() + # + except Exception as e: + LOGGER.error("error adding label to an output", error=e) + return SimpleErrorResponse( + message="error adding label to an output" + ).model_dump() + + +@transactions_page.route("/outputs/label", methods=["DELETE"]) +@inject +def remove_output_label( + wallet_service: WalletService = Provide[ServiceContainer.wallet_service], +): + """ + Remove a label from a specific output (defined by the txid and vout). + """ + try: + remove_output_label_request_dto = RemoveOutputLabelRequestDto.model_validate( + dict( + txid=request.args.get("txid"), + vout=request.args.get("vout"), + labelName=request.args.get("labelName"), + ) + ) + labels = wallet_service.remove_label_from_output( + remove_output_label_request_dto.txid, + remove_output_label_request_dto.vout, + remove_output_label_request_dto.labelName, + ) + + return RemoveOutputLabelResponseDto.model_validate( + dict( + labels=[ + dict( + label=label.name, + display_name=label.display_name, + description=label.description, + ) + for label in labels + ] + ) + ).model_dump() + + except Exception as e: + LOGGER.error("Error removing a label from an output.", error=e) + return SimpleErrorResponse( + message="Error removing a label from an output." + ).model_dump() diff --git a/backend/src/database.py b/backend/src/database.py index 0fd25cd0..e2e664d9 100644 --- a/backend/src/database.py +++ b/backend/src/database.py @@ -7,3 +7,27 @@ class Base(DeclarativeBase): DB = SQLAlchemy() + + +def populate_labels(): + try: + from src.models.label import Label, LabelName, label_descriptions + + for label_name in LabelName: + # Check if the label already exists + existing_label = DB.session.query(Label).filter_by(name=label_name).first() + + # If it does not exist, create a new one + if not existing_label: + label = Label( + name=label_name, + display_name=label_name.value, + description=label_descriptions[label_name], + ) + DB.session.add(label) + + # Commit the session only once after all additions + DB.session.commit() + except Exception as e: + DB.session.rollback() + raise e diff --git a/backend/src/models/label.py b/backend/src/models/label.py new file mode 100644 index 00000000..173fc432 --- /dev/null +++ b/backend/src/models/label.py @@ -0,0 +1,33 @@ +from sqlalchemy import String, Enum, Integer +from enum import Enum as PyEnum +from .output_labels import output_labels + + +from src.database import DB + + +class LabelName(PyEnum): + DO_NOT_SPEND = "do not spend" + KYCED = "kyced" + NOT_KYCED = "not kyced" + # Add more labels as needed + + +label_descriptions = { + LabelName.DO_NOT_SPEND: "This output should not be spent. This can be helpful for outputs that may comprimise your privacy, like KYCED outputs, or toxic change from a coinjoin.", + LabelName.KYCED: "KYC, also known as Know Your Customer, is the process of verifying the identity of customers. This output has been KYCed, which means it is attached to your name, and therefore you should be careful when spending it as it may be being monitoring.", + LabelName.NOT_KYCED: "KYC, also known as Know Your Customer, is the process of verifying the identity of customers. This output has not been KYCed, and therefore is not attached to your name, increasing your privacy.", +} + + +class Label(DB.Model): + __tablename__ = "labels" # Specify the table name + + id = DB.Column(Integer, primary_key=True, autoincrement=True) + name = DB.Column(Enum(LabelName), unique=True, nullable=False) + display_name = DB.Column(String, unique=True, nullable=False) + description = DB.Column(String, unique=True, nullable=False) + + outputs = DB.relationship( + "Output", secondary=output_labels, back_populates="labels" + ) diff --git a/backend/src/models/output_labels.py b/backend/src/models/output_labels.py new file mode 100644 index 00000000..1aca7fe2 --- /dev/null +++ b/backend/src/models/output_labels.py @@ -0,0 +1,25 @@ +# from sqlalchemy import Table, ForeignKey, Integer +# +# from src.database import DB +# +# +# class OutputLabel(DB.Model): +# __tablename__ = "output_labels" +# id = DB.Column(Integer, primary_key=True) +# output_id = DB.Column( +# "output_id", DB.String, ForeignKey("outputs.id"), primary_key=True +# ) +# label_id = DB.Column( +# "label_id", DB.String, ForeignKey("labels.id"), primary_key=True +# ) + + +from sqlalchemy import Table, Column, String, ForeignKey +from src.database import DB + +output_labels = Table( + "output_labels", + DB.Model.metadata, + Column("output_id", String, ForeignKey("outputs.id"), primary_key=True), + Column("label_id", String, ForeignKey("labels.id"), primary_key=True), +) diff --git a/backend/src/models/outputs.py b/backend/src/models/outputs.py new file mode 100644 index 00000000..498b4b50 --- /dev/null +++ b/backend/src/models/outputs.py @@ -0,0 +1,23 @@ +from sqlalchemy import Integer +from src.database import DB +import uuid +from .output_labels import output_labels + + +class Output(DB.Model): + __tablename__ = "outputs" # Specify the table name + + # Auto-incrementing integer + id = DB.Column(Integer, primary_key=True, autoincrement=True) + + txid = DB.Column( + DB.String(), + nullable=False, + ) + vout = DB.Column(DB.Integer, nullable=False, default=0) + + # Relationship to labels + labels = DB.relationship("Label", secondary=output_labels, back_populates="outputs") + + # Unique constraint on the combination of txid and vout + __table_args__ = (DB.UniqueConstraint("txid", "vout", name="uq_txid_vout"),) diff --git a/backend/src/my_types/__init__.py b/backend/src/my_types/__init__.py index f0311cdd..b77a4cd6 100644 --- a/backend/src/my_types/__init__.py +++ b/backend/src/my_types/__init__.py @@ -8,6 +8,11 @@ GetAllUtxosResponseDto, GetAllTransactionsResponseDto, GetAllOutputsResponseDto, + AddOutputLabelRequestDto, + RemoveOutputLabelRequestDto, + AddOutputLabelResponseDto, + RemoveOutputLabelResponseDto, + GetOutputLabelsResponseDto, ) from src.my_types.controller_types.fees_dtos import GetCurrentFeesResponseDto diff --git a/backend/src/my_types/controller_types/utxos_dtos.py b/backend/src/my_types/controller_types/utxos_dtos.py index 116c518b..ba8b8093 100644 --- a/backend/src/my_types/controller_types/utxos_dtos.py +++ b/backend/src/my_types/controller_types/utxos_dtos.py @@ -53,6 +53,12 @@ class OutputDto(BaseModel): spending_index_n: Optional[int] +class OutputDetailDto(OutputDto): + annominity_set: Optional[int] + txid: Optional[str] + labels: Optional[List[str]] + + class TransactionDetailDto(BaseModel): txid: str date: Optional[str] @@ -121,4 +127,34 @@ class GetAllTransactionsResponseDto(BaseModel): class GetAllOutputsResponseDto(BaseModel): - outputs: list[OutputDto] + outputs: list[OutputDetailDto] + + +class AddOutputLabelRequestDto(BaseModel): + txid: str + vout: int + labelName: str + + +class RemoveOutputLabelRequestDto(BaseModel): + txid: str + vout: int + labelName: str + + +class OutputLabelDto(BaseModel): + label: str + display_name: str + description: str + + +class RemoveOutputLabelResponseDto(BaseModel): + labels: list[OutputLabelDto] + + +class AddOutputLabelResponseDto(BaseModel): + labels: list[OutputLabelDto] + + +class GetOutputLabelsResponseDto(BaseModel): + labels: list[OutputLabelDto] diff --git a/backend/src/my_types/transactions.py b/backend/src/my_types/transactions.py new file mode 100644 index 00000000..da9917a7 --- /dev/null +++ b/backend/src/my_types/transactions.py @@ -0,0 +1,30 @@ +from typing import List, Any, Optional +from bitcoinlib.transactions import Output + + +class LiveWalletOutput(Output): + def __init__( + self, + annominity_set: int = 1, + txid: Optional[str] = None, + base_output: Optional[Output] = None, + labels: Optional[List[str]] = None, + ): + if labels is None: + labels = [] + + for key, value in vars(base_output).items(): + setattr(self, key, value) # Set each attribute in the subclass + self.annominity_set = annominity_set + self.txid = txid + self.labels = labels + + def as_dict(self) -> dict[str, Any]: + # Get the dictionary from the base class + base_dict = super().as_dict() + + # Add additional features to the dictionary + base_dict["annominity_set"] = self.annominity_set + base_dict["txid"] = self.txid + base_dict["labels"] = self.labels + return base_dict diff --git a/backend/src/services/wallet/wallet.py b/backend/src/services/wallet/wallet.py index 321fdb6c..54b9204d 100644 --- a/backend/src/services/wallet/wallet.py +++ b/backend/src/services/wallet/wallet.py @@ -1,6 +1,8 @@ from dataclasses import dataclass import bdkpython as bdk from bitcoinlib.transactions import Output, Transaction +from src.models.label import LabelName, Label +from src.models.outputs import Output as OutputModel from typing import Any, Literal, Optional, cast, List from src.api import electrum_request, parse_electrum_url @@ -15,6 +17,8 @@ FeeDetails, GetUtxosRequestDto, ) +from src.my_types.controller_types.utxos_dtos import OutputLabelDto +from src.my_types.transactions import LiveWalletOutput from src.services.wallet.raw_output_script_examples import ( p2pkh_raw_output_script, p2pk_raw_output_script, @@ -303,18 +307,109 @@ def get_all_transactions( all_tx_details.append(electrum_response.data) return all_tx_details - def get_all_outputs(self) -> List[Output]: - """Get all spent and unspent transaction outputs for the current wallet.""" + def get_all_outputs(self) -> List[LiveWalletOutput]: + """Get all spent and unspent transaction outputs for the current wallet and mutate them as needed. + Calculate the annominity set for each output. + Attach the txid to each output. + Attach all labels to each output. + Sync the database with the incoming outputs. + """ all_transactions = self.get_all_transactions() - all_outputs: List[Output] = [] + all_outputs: List[LiveWalletOutput] = [] for transaction in all_transactions: + annominity_sets = self.calculate_output_annominity_sets(transaction.outputs) for output in transaction.outputs: + db_output = self.sync_local_db_with_incoming_output( + txid=transaction.txid, output=output + ) script = bdk.Script(output.script.raw) if self.wallet and self.wallet.is_mine(script): - all_outputs.append(output) + annominity_set = annominity_sets.get(output.value, 1) + + extended_output = LiveWalletOutput( + annominity_set=annominity_set, + base_output=output, + txid=transaction.txid, + labels=[label.display_name for label in db_output.labels], + ) + + all_outputs.append(extended_output) return all_outputs + def sync_local_db_with_incoming_output( + self, txid: str, output: Output + ) -> OutputModel: + """Sync the local database with the incoming output. + + If the output is not in the database, add it. + """ + + db_output = OutputModel.query.filter_by(txid=txid, vout=output.output_n).first() + if not db_output: + db_output = self.add_output_to_db(txid=txid, output=output) + return db_output + + def add_output_to_db(self, output: Output, txid: str) -> OutputModel: + db_output = OutputModel(txid=txid, vout=output.output_n, labels=[]) + DB.session.add(db_output) + DB.session.commit() + return db_output + + def calculate_output_annominity_sets( + self, transaction_outputs: List[Output] + ) -> dict[str, int]: # -> {"value": count } + """Calculate the annominity set for a given output in a transaction. + + The annominity set is the number of outputs with the same btc value in a transaction. + """ + # loop through the transaction outputs and + # count how many equal outputs there are for each amount + # {"value": equal_output_count} + output_count = {} + for output in transaction_outputs: + if output_count.get(output.value): + output_count[output.value] += 1 + else: + output_count[output.value] = 1 + return output_count + + def get_output_labels(self) -> List[OutputLabelDto]: + """Get all the labels for the outputs in the wallet.""" + labels = Label.query.all() + return [ + OutputLabelDto( + label=label.name, + display_name=label.display_name, + description=label.description, + ) + for label in labels + ] + + # TODO should this even go here + def add_label_to_output( + self, txid: str, vout: int, label_display_name: str + ) -> list[Label]: + """Add a label to an output in the db.""" + db_output = OutputModel.query.filter_by(txid=txid, vout=vout).first() + label = Label.query.filter_by(display_name=label_display_name).first() + db_output.labels.append(label) + DB.session.commit() + return db_output.labels + + def remove_label_from_output( + self, txid: str, vout: int, label_display_name: str + ) -> list[Label]: + """Remove a label from an output in the db.""" + db_output = OutputModel.query.filter_by(txid=txid, vout=vout).first() + label = Label.query.filter_by(display_name=label_display_name).first() + # Remove the label from the output's labels collection + if label in db_output.labels: + db_output.labels.remove(label) + DB.session.commit() + + return db_output.labels + def get_utxos_info(self, utxos_wanted: List[bdk.OutPoint]) -> List[bdk.LocalUtxo]: """For a given set of txids and the vout pointing to a utxo, return the utxos""" existing_utxos = cast(List[bdk.LocalUtxo], self.get_all_utxos()) diff --git a/src/app/api/api.ts b/src/app/api/api.ts index 3d59a7f1..9d94ec05 100644 --- a/src/app/api/api.ts +++ b/src/app/api/api.ts @@ -17,6 +17,11 @@ import { GetBTCPriceResponseType, GetTransactionsResponseType, GetOutputsResponseType, + GetOutputLabelsResponseType, + AddLabelRequestBody, + RemoveLabelRequestParams, + AddLabelResponseType, + RemoveLabelResponseType, } from './types'; import { Network } from '../types/network'; @@ -76,6 +81,43 @@ export class ApiClient { return data as GetOutputsResponseType; } + + static async getOutputLabels() { + const response = + await fetchHandler(`${configs.backendServerBaseUrl}/transactions/outputs/labels +`); + + const data = await response.json(); + + return data as GetOutputLabelsResponseType; + } + static async addOutputLabel(body: AddLabelRequestBody) { + const response = await fetchHandler( + `${configs.backendServerBaseUrl}/transactions/outputs/label`, + 'POST', + body, + ); + + const data = await response.json(); + + return data as AddLabelResponseType; + } + + static async removeOutputLabel( + txid: RemoveLabelRequestParams["txid"], + vout: RemoveLabelRequestParams["vout"], + labelName: RemoveLabelRequestParams["labelName"], + ) { + const response = await fetchHandler( + `${configs.backendServerBaseUrl}/transactions/outputs/label?txid=${txid}&vout=${vout}&labelName=${labelName}`, + 'DELETE', + ); + + const data = await response.json(); + + return data as RemoveLabelResponseType; + } + static async createTxFeeEstimation( utxos: UtxoRequestParam[], feeRate: number = 1, diff --git a/src/app/api/types.ts b/src/app/api/types.ts index 0ce0d974..1af1028d 100644 --- a/src/app/api/types.ts +++ b/src/app/api/types.ts @@ -104,7 +104,7 @@ export type GetBTCPriceResponseType = { JPY: number; }; -type TransactionInputType = { +export type TransactionInputType = { index_n: number; prev_txid: string; output_n: number; @@ -132,7 +132,7 @@ type TransactionInputType = { valid?: boolean; }; -type TransactionOutputType = { +export type TransactionOutputType = { value: number; // in sats script: string; script_type: string; // e.g., "p2wpkh" @@ -143,6 +143,18 @@ type TransactionOutputType = { spent: boolean; spending_txid: string; spending_index_n?: number; + txid: string; + annominity_set: number; + labels: string[]; +}; + +export type OutputLabelType = { + label: string; + display_name: string; + description: string; +}; +export type GetOutputLabelsResponseType = { + labels: [OutputLabelType]; }; export type GetTransactionsResponseType = { @@ -174,3 +186,23 @@ export type GetTransactionsResponseType = { export type GetOutputsResponseType = { outputs: TransactionOutputType[]; }; + +export type AddLabelRequestBody = { + txid: string; + vout: number; + labelName: string; +}; + +export type AddLabelResponseType = { + labels: [OutputLabelType] +}; + +export type RemoveLabelRequestParams = { + txid: string; + vout: number; + labelName: string; +}; + +export type RemoveLabelResponseType = { + labels: [OutputLabelType] +}; diff --git a/src/app/components/OutputModal.tsx b/src/app/components/OutputModal.tsx new file mode 100644 index 00000000..7dd7e223 --- /dev/null +++ b/src/app/components/OutputModal.tsx @@ -0,0 +1,95 @@ +import { Modal, Chip, Tooltip } from '@mantine/core'; +import { useState } from 'react'; +import { BtcMetric, btcSatHandler } from '../types/btcSatHandler'; +import { OutputLabelType, TransactionOutputType } from '../api/types'; +import { useAddUtxoLabel, useRemoveUtxoLabel } from '../hooks/transactions'; + +type OutputModalProps = { + output: TransactionOutputType; + btcMetric: BtcMetric; + opened: boolean; + onClose: () => void; + labels: OutputLabelType[]; +}; +export const OutputModal = ({ + output, + btcMetric, + opened, + onClose, + labels, +}: OutputModalProps) => { + const [selectedChips, setSelectedChips] = useState(output.labels); + const addUtxoLabel = useAddUtxoLabel(); + const removeUtxoLabel = useRemoveUtxoLabel(); + const amount = btcSatHandler( + Number(output.value).toFixed(2).toLocaleString(), + btcMetric, + ); + + const handleChipChange = (label: OutputLabelType, isChecked: boolean) => { + if (isChecked) { + addUtxoLabel.mutate({ + txid: output.txid, + vout: output.output_n, + labelName: label.display_name, + }); + } else { + removeUtxoLabel.mutate({ + txid: output.txid, + vout: output.output_n, + labelName: label.display_name, + }); + } + }; + + return ( + +
+

address: {output.address}

+

+ amount:{' '} + {btcMetric === BtcMetric.BTC + ? amount + : Number(amount).toLocaleString()} +

+

Annominity set: {output.annominity_set}

+

v out: {output.output_n}

+

hash: {output.public_hash}

+

script: {output.script}

+

script type: {output.script_type}

+

Recieved txid: {output.txid}

+

is spent?{output.spent ? 'Yes' : 'No'}

+

spending index: {output.spending_index_n}

+

spending txid: {output.spending_txid}

+ labels here + +
+ {labels.map((label) => ( + + handleChipChange(label, isChecked)} + width={30} + value={label.display_name} + > + {label.display_name} + + + ))} +
+ +
+ + ); +}; diff --git a/src/app/components/privacy/txosTable.tsx b/src/app/components/privacy/txosTable.tsx index 89abdfbd..bfea1c58 100644 --- a/src/app/components/privacy/txosTable.tsx +++ b/src/app/components/privacy/txosTable.tsx @@ -1,4 +1,4 @@ -import React, { useMemo } from 'react'; +import React, { useMemo, useState } from 'react'; import { createTheme, ThemeProvider } from '@mui/material'; import { @@ -6,14 +6,7 @@ import { useMaterialReactTable, } from 'material-react-table'; -import { - CopyButton, - rem, - Tooltip, - ActionIcon, - Chip, - ChipGroup, -} from '@mantine/core'; +import { CopyButton, rem, Tooltip, ActionIcon, Chip } from '@mantine/core'; import { MdLabelOutline } from 'react-icons/md'; import { @@ -21,17 +14,24 @@ import { IconCircleCheck, IconCircleX, IconCopy, + IconEdit, } from '@tabler/icons-react'; import { BtcMetric, btcSatHandler } from '../../types/btcSatHandler'; +import { OutputModal } from '../OutputModal'; +import { TransactionOutputType } from '../../api/types'; +import { useGetOutputLabels } from '../../hooks/transactions'; const sectionColor = 'rgb(1, 67, 97)'; type TxosTableProps = { - txos: any; + txos: TransactionOutputType[]; btcMetric: BtcMetric; }; export const TxosTable = ({ txos, btcMetric }: TxosTableProps) => { + const getOutputLabelsQuery = useGetOutputLabels(); + const [isOutputModalShowing, setIsOutputModalShowing] = useState(false); + const [selectedOutput, setSelectedOutput] = useState(); const columns = useMemo(() => { const defaultColumns = [ { @@ -46,7 +46,7 @@ export const TxosTable = ({ txos, btcMetric }: TxosTableProps) => { const abrv = `${prefix}....${suffix}`; return (
- +

{abrv}

 @@ -94,29 +94,57 @@ export const TxosTable = ({ txos, btcMetric }: TxosTableProps) => { ); }, }, + { + header: 'Annominity', + accessorKey: 'annominity_set', + size: 30, + Cell: ({ row }) => { + return ( +
+

{row.original.annominity_set}

+
+ ); + }, + }, { header: 'Labels', - accessorKey: 'label', + accessorKey: 'labels', size: 250, - Cell: () => { - // mock labels for now - const allLabels = ['do not spend', 'bad change', "another"]; + Cell: ({ row }) => { + const allLabels = row.original.labels || []; return (
- {allLabels.map((label) => ( - } - color="red" - checked={true} - variant="light" - className="mr-1" - classNames={{ - checkIcon: 'h-0 hidden', - }} - > - {label} - - ))} + {allLabels.length > 0 ? ( + allLabels.map((label) => ( + } + color="red" + checked={true} + variant="light" + className="mr-1" + > + {label} + + )) + ) : ( +

None

+ )} + + { + setSelectedOutput(row.original); + setIsOutputModalShowing(true); + }} + variant="outline" + aria-label="edit" + data-testid="edit-label-button" + color="gray" + size="sm" + className="mt-auto ml-1" + > + +
); }, @@ -141,7 +169,7 @@ export const TxosTable = ({ txos, btcMetric }: TxosTableProps) => { ]; return defaultColumns; - }, [txos, btcMetric]); + }, [txos, btcMetric, isOutputModalShowing]); const table = useMaterialReactTable({ columns, @@ -164,6 +192,7 @@ export const TxosTable = ({ txos, btcMetric }: TxosTableProps) => { enableTopToolbar: true, positionToolbarAlertBanner: 'none', positionToolbarDropZone: 'top', + enableEditing: false, renderTopToolbarCustomActions: ({ table }) => { return (
@@ -231,6 +260,15 @@ export const TxosTable = ({ txos, btcMetric }: TxosTableProps) => { > + {isOutputModalShowing && ( + setIsOutputModalShowing(false)} + labels={getOutputLabelsQuery.data?.labels || []} + /> + )}
); }; diff --git a/src/app/hooks/transactions.ts b/src/app/hooks/transactions.ts new file mode 100644 index 00000000..1dd43a1a --- /dev/null +++ b/src/app/hooks/transactions.ts @@ -0,0 +1,63 @@ +import { ApiClient } from '../api/api'; +import { useMutation, useQuery } from 'react-query'; +import { AddLabelRequestBody, RemoveLabelRequestParams } from '../api/types'; + +export const uxtoQueryKeys = { + getTransactions: ['getTransactions'], + getOutputs: ['getOutputs'], + getOutputLabels: ['getOutputLabels'], +}; + +export function useGetTransactions() { + return useQuery( + uxtoQueryKeys.getTransactions, + () => ApiClient.getTransactions(), + { + refetchOnWindowFocus: true, + }, + ); +} + +export function useGetOutputs() { + return useQuery(uxtoQueryKeys.getOutputs, () => ApiClient.getOutputs(), { + refetchOnWindowFocus: true, + }); +} + +export function useGetOutputLabels() { + return useQuery( + uxtoQueryKeys.getOutputLabels, + () => ApiClient.getOutputLabels(), + { + refetchOnWindowFocus: false, + }, + ); +} + +export function useAddUtxoLabel(onError?: () => void) { + return useMutation( + (addLabelRequestBody: AddLabelRequestBody) => + ApiClient.addOutputLabel(addLabelRequestBody), + { + onError: () => { + onError(); + }, + }, + ); +} + +export function useRemoveUtxoLabel(onError?: () => void) { + return useMutation( + (requestParams: RemoveLabelRequestParams) => + ApiClient.removeOutputLabel( + requestParams.txid, + requestParams.vout, + requestParams.labelName, + ), + { + onError: () => { + onError(); + }, + }, + ); +} diff --git a/src/app/hooks/utxos.ts b/src/app/hooks/utxos.ts index 9753b578..6ccf44fc 100644 --- a/src/app/hooks/utxos.ts +++ b/src/app/hooks/utxos.ts @@ -5,9 +5,7 @@ import { useMutation, useQuery } from 'react-query'; export const uxtoQueryKeys = { getBalance: ['getBalance'], getUtxos: ['getUtxos'], - getTransactions: ['getTransactions'], getCurrentFees: ['getCurrentFees'], - getOutputs: ['getOutputs'], }; export function useGetBalance() { @@ -21,21 +19,7 @@ export function useGetUtxos() { }); } -export function useGetTransactions() { - return useQuery( - uxtoQueryKeys.getTransactions, - () => ApiClient.getTransactions(), - { - refetchOnWindowFocus: true, - }, - ); -} -export function useGetOutputs() { - return useQuery(uxtoQueryKeys.getOutputs, () => ApiClient.getOutputs(), { - refetchOnWindowFocus: true, - }); -} export function useCreateTxFeeEstimate( utxos: UtxoRequestParam[], diff --git a/src/app/pages/Privacy.tsx b/src/app/pages/Privacy.tsx index 13d86f0c..0b88354c 100644 --- a/src/app/pages/Privacy.tsx +++ b/src/app/pages/Privacy.tsx @@ -1,6 +1,6 @@ import { Tabs, rem } from '@mantine/core'; import { IconCoins, IconArrowsDownUp, IconEye } from '@tabler/icons-react'; -import { useGetTransactions, useGetOutputs } from '../hooks/utxos'; +import { useGetTransactions, useGetOutputs } from '../hooks/transactions'; import { TxosTable } from '../components/privacy/txosTable'; import { BtcMetric } from '../types/btcSatHandler'; From e56036e3ff023f0ed982de62762a6c3bc3b11e3c Mon Sep 17 00:00:00 2001 From: Joseph Wyman Date: Sat, 26 Oct 2024 21:44:11 -0400 Subject: [PATCH 14/85] add saving labels in save file, and then prepopulating the db when file wallet is loaded and logged in --- backend/src/app.py | 6 +- backend/src/controllers/transactions.py | 56 +++++++- backend/src/controllers/wallet.py | 3 +- backend/src/database.py | 3 +- backend/src/my_types/__init__.py | 3 + .../my_types/controller_types/utxos_dtos.py | 16 ++- backend/src/my_types/transactions.py | 5 + backend/src/services/wallet/wallet.py | 123 +++++++++++++++--- src/app/api/api.ts | 32 ++++- src/app/api/types.ts | 14 +- src/app/components/privacy/txosTable.tsx | 23 +++- src/app/hooks/transactions.ts | 66 +++++++++- src/app/pages/Home.tsx | 29 +++++ src/app/pages/Privacy.tsx | 5 +- src/app/pages/WalletSignIn.tsx | 2 + src/app/types/wallet.ts | 2 + src/main/main.ts | 6 + src/main/preload.ts | 3 +- 18 files changed, 358 insertions(+), 39 deletions(-) diff --git a/backend/src/app.py b/backend/src/app.py index 6bb6bb42..915a81e3 100644 --- a/backend/src/app.py +++ b/backend/src/app.py @@ -93,6 +93,9 @@ def setup_database(app): app.config["SQLALCHEMY_TRACK_MODIFICATIONS"] = False DB.init_app(app) with app.app_context(): + # Drop all existing tables + DB.drop_all() # Clear the database, incase anything was left over. + DB.create_all() populate_labels() @@ -105,7 +108,8 @@ def setup_database(app): if is_testing is False: # hwi will fail on macos unless it is run in a single thread, threrefore set threaded to False - app.run(host="127.0.0.1", port=5011, debug=is_development, threaded=False) + app.run(host="127.0.0.1", port=5011, + debug=is_development, threaded=False) else: # this will run when the app is run from the generated executable # which is done in the production app. diff --git a/backend/src/controllers/transactions.py b/backend/src/controllers/transactions.py index e1e9a6a5..94c7d091 100644 --- a/backend/src/controllers/transactions.py +++ b/backend/src/controllers/transactions.py @@ -14,10 +14,14 @@ RemoveOutputLabelResponseDto, AddOutputLabelResponseDto, GetOutputLabelsResponseDto, + GetOutputLabelsUniqueResponseDto, + PopulateOutputLabelsUniqueResponseDto, + PopulateOutputLabelsUniqueRequestDto, ) from src.my_types.controller_types.generic_response_types import SimpleErrorResponse -transactions_page = Blueprint("get_transactions", __name__, url_prefix="/transactions") +transactions_page = Blueprint( + "get_transactions", __name__, url_prefix="/transactions") LOGGER = structlog.get_logger() @@ -34,7 +38,8 @@ def get_txos( transactions = wallet_service.get_all_transactions() return GetAllTransactionsResponseDto.model_validate( - dict(transactions=[transaction.as_dict() for transaction in transactions]) + dict(transactions=[transaction.as_dict() + for transaction in transactions]) ).model_dump() except Exception as e: @@ -84,6 +89,53 @@ def get_output_labels( ).model_dump() +# TODO better name +@transactions_page.route("/outputs/labels-unique", methods=["GET"]) +@inject +def get_output_labels_unique( + wallet_service: WalletService = Provide[ServiceContainer.wallet_service], +): + """ + Get all labels for each txid-vout + """ + try: + labels = wallet_service.get_output_labels_unique() + + return GetOutputLabelsUniqueResponseDto.model_validate(labels).model_dump() + except Exception as e: + LOGGER.error("error getting all output labels unique", error=e) + return SimpleErrorResponse( + message="error getting all output labels unique" + ).model_dump() + + +# TODO better name for endpoint +@transactions_page.route("/outputs/labels-unique", methods=["POST"]) +@inject +def populate_output_labels_unique( + wallet_service: WalletService = Provide[ServiceContainer.wallet_service], +): + """ + Populate all labels and outputs for each txid-vout + + This is used when a user loads a wallet and needs to populate the labels + for their wallet's outputs that they have saved. + """ + try: + data = PopulateOutputLabelsUniqueRequestDto(json.loads(request.data)) + wallet_service.populate_outputs_and_labels(data) + + return PopulateOutputLabelsUniqueResponseDto.model_validate( + {"success": True} + ).model_dump() + # + except Exception as e: + LOGGER.error("error populating all labels and outputs.", error=e) + return SimpleErrorResponse( + message="error populating all labels and outputs." + ).model_dump() + + @transactions_page.route("/outputs/label", methods=["POST"]) @inject def add_output_label( diff --git a/backend/src/controllers/wallet.py b/backend/src/controllers/wallet.py index 45cbc685..6839d028 100644 --- a/backend/src/controllers/wallet.py +++ b/backend/src/controllers/wallet.py @@ -114,9 +114,10 @@ def delete_wallet(): """ try: WalletService.remove_global_wallet_and_details() + WalletService.remove_output_and_related_label_data() return DeleteWalletResponseDto( - message="wallet successfully deleted", + message="wallet and related data successfully deleted", ).model_dump() except ValidationError as e: diff --git a/backend/src/database.py b/backend/src/database.py index e2e664d9..bebdcdc3 100644 --- a/backend/src/database.py +++ b/backend/src/database.py @@ -15,7 +15,8 @@ def populate_labels(): for label_name in LabelName: # Check if the label already exists - existing_label = DB.session.query(Label).filter_by(name=label_name).first() + existing_label = DB.session.query( + Label).filter_by(name=label_name).first() # If it does not exist, create a new one if not existing_label: diff --git a/backend/src/my_types/__init__.py b/backend/src/my_types/__init__.py index b77a4cd6..eb0535b4 100644 --- a/backend/src/my_types/__init__.py +++ b/backend/src/my_types/__init__.py @@ -13,6 +13,9 @@ AddOutputLabelResponseDto, RemoveOutputLabelResponseDto, GetOutputLabelsResponseDto, + GetOutputLabelsUniqueResponseDto, + PopulateOutputLabelsUniqueResponseDto, + PopulateOutputLabelsUniqueRequestDto, ) from src.my_types.controller_types.fees_dtos import GetCurrentFeesResponseDto diff --git a/backend/src/my_types/controller_types/utxos_dtos.py b/backend/src/my_types/controller_types/utxos_dtos.py index ba8b8093..698cf9ad 100644 --- a/backend/src/my_types/controller_types/utxos_dtos.py +++ b/backend/src/my_types/controller_types/utxos_dtos.py @@ -1,5 +1,5 @@ -from typing import Optional, List -from pydantic import BaseModel, field_validator, Field +from typing import Optional, List, Dict +from pydantic import BaseModel, RootModel, field_validator, Field import structlog LOGGER = structlog.get_logger() @@ -158,3 +158,15 @@ class AddOutputLabelResponseDto(BaseModel): class GetOutputLabelsResponseDto(BaseModel): labels: list[OutputLabelDto] + + +class GetOutputLabelsUniqueResponseDto(RootModel): + root: Dict[str, List[OutputLabelDto]] + + +class PopulateOutputLabelsUniqueRequestDto(RootModel): + root: Dict[str, List[OutputLabelDto]] + + +class PopulateOutputLabelsUniqueResponseDto(BaseModel): + success: bool diff --git a/backend/src/my_types/transactions.py b/backend/src/my_types/transactions.py index da9917a7..19433179 100644 --- a/backend/src/my_types/transactions.py +++ b/backend/src/my_types/transactions.py @@ -3,6 +3,11 @@ class LiveWalletOutput(Output): + """ + A bitcoin output which extendes the bitcoinlib Output and adds additional + fields unique to live wallet, like annominity_set, txid and labels. + """ + def __init__( self, annominity_set: int = 1, diff --git a/backend/src/services/wallet/wallet.py b/backend/src/services/wallet/wallet.py index 54b9204d..516f0ca7 100644 --- a/backend/src/services/wallet/wallet.py +++ b/backend/src/services/wallet/wallet.py @@ -1,9 +1,10 @@ from dataclasses import dataclass import bdkpython as bdk from bitcoinlib.transactions import Output, Transaction -from src.models.label import LabelName, Label +from sqlalchemy import func +from src.models.label import Label from src.models.outputs import Output as OutputModel -from typing import Any, Literal, Optional, cast, List +from typing import Literal, Optional, cast, List, Dict from src.api import electrum_request, parse_electrum_url from src.api.electrum import ( @@ -17,7 +18,10 @@ FeeDetails, GetUtxosRequestDto, ) -from src.my_types.controller_types.utxos_dtos import OutputLabelDto +from src.my_types.controller_types.utxos_dtos import ( + OutputLabelDto, + PopulateOutputLabelsUniqueRequestDto, +) from src.my_types.transactions import LiveWalletOutput from src.services.wallet.raw_output_script_examples import ( p2pkh_raw_output_script, @@ -89,6 +93,24 @@ def create_wallet( DB.session.add(new_wallet) DB.session.commit() + @classmethod + def remove_output_and_related_label_data(cls): + # make this reusable since it is used twice? + outputs_with_label = ( + DB.session.query(OutputModel) + .join(OutputModel.labels) + .group_by(OutputModel.id) + .having(func.count(Label.id) > 0) + .all() + ) + # remove all rows in the output_labels table + for output in outputs_with_label: + output.labels = [] + DB.session.flush() + # remove all rows in the OutputModel table + DB.session.query(OutputModel).delete() + DB.session.commit() + @classmethod def remove_global_wallet_and_details(cls): DB.session.query(Wallet).delete() @@ -139,7 +161,8 @@ def connect_wallet( ) wallet_change_descriptor = ( - bdk.Descriptor(change_descriptor, bdk.Network._value2member_map_[network]) + bdk.Descriptor(change_descriptor, + bdk.Network._value2member_map_[network]) if change_descriptor else None ) @@ -161,7 +184,8 @@ def connect_wallet( database_config=db_config, ) - LOGGER.info(f"Connecting a new wallet to electrum server {wallet_details_id}") + LOGGER.info( + f"Connecting a new wallet to electrum server {wallet_details_id}") LOGGER.info(f"xpub {wallet_descriptor.as_string()}") wallet.sync(blockchain, None) @@ -210,7 +234,8 @@ def create_spendable_descriptor( twelve_word_secret = bdk.Mnemonic(bdk.WordCount.WORDS12) # xpriv - descriptor_secret_key = bdk.DescriptorSecretKey(network, twelve_word_secret, "") + descriptor_secret_key = bdk.DescriptorSecretKey( + network, twelve_word_secret, "") wallet_descriptor = None if script_type == ScriptType.P2PKH: @@ -317,10 +342,11 @@ def get_all_outputs(self) -> List[LiveWalletOutput]: all_transactions = self.get_all_transactions() all_outputs: List[LiveWalletOutput] = [] for transaction in all_transactions: - annominity_sets = self.calculate_output_annominity_sets(transaction.outputs) + annominity_sets = self.calculate_output_annominity_sets( + transaction.outputs) for output in transaction.outputs: db_output = self.sync_local_db_with_incoming_output( - txid=transaction.txid, output=output + txid=transaction.txid, vout=output.output_n ) script = bdk.Script(output.script.raw) if self.wallet and self.wallet.is_mine(script): @@ -338,20 +364,22 @@ def get_all_outputs(self) -> List[LiveWalletOutput]: return all_outputs def sync_local_db_with_incoming_output( - self, txid: str, output: Output + self, + txid: str, + vout: int, ) -> OutputModel: """Sync the local database with the incoming output. If the output is not in the database, add it. """ - db_output = OutputModel.query.filter_by(txid=txid, vout=output.output_n).first() + db_output = OutputModel.query.filter_by(txid=txid, vout=vout).first() if not db_output: - db_output = self.add_output_to_db(txid=txid, output=output) + db_output = self.add_output_to_db(txid=txid, vout=vout) return db_output - def add_output_to_db(self, output: Output, txid: str) -> OutputModel: - db_output = OutputModel(txid=txid, vout=output.output_n, labels=[]) + def add_output_to_db(self, vout: int, txid: str) -> OutputModel: + db_output = OutputModel(txid=txid, vout=vout, labels=[]) DB.session.add(db_output) DB.session.commit() return db_output @@ -386,13 +414,77 @@ def get_output_labels(self) -> List[OutputLabelDto]: for label in labels ] - # TODO should this even go here + # TODO name this better + def get_output_labels_unique( + self, + ) -> Dict[str, OutputLabelDto]: + """Get all the labels for the outputs in the wallet + and return them as a dictionary of the key-id + mapped to an array of labels. + """ + outputs_with_label = ( + DB.session.query(OutputModel) + .join(OutputModel.labels) + .group_by(OutputModel.id) + .having(func.count(Label.id) > 0) + .all() + ) + + result = {} + + for output in outputs_with_label: + for label in output.labels: + key = f"{output.txid}-{output.vout}" + if result.get(key, None) is None: + result[key] = [ + OutputLabelDto( + label=label.name, + display_name=label.display_name, + description=label.description, + ) + ] + else: + result[key].append( + OutputLabelDto( + label=label.name, + display_name=label.display_name, + description=label.description, + ) + ) + + return result + + def populate_outputs_and_labels( + self, unique_output_labels: PopulateOutputLabelsUniqueRequestDto + ) -> None: # TODO maybe a success of fail reutn type? + try: + model_dump = unique_output_labels.model_dump() + for unique_output_txid_vout in model_dump.keys(): + txid, vout = unique_output_txid_vout.split("-") + # db_output = OutputModel.query.filter_by(txid=txid, vout=vout).first() + # if db_output is None: + # # add the output to the db + # db_output = OutputModel(txid=txid, vout=vout, labels=[]) + # DB.session.add(db_output) + # DB.session.flush() + self.sync_local_db_with_incoming_output(txid, vout) + output_labels = model_dump[unique_output_txid_vout] + for label in output_labels: + display_name = label["display_name"] + self.add_label_to_output(txid, vout, display_name) + except Exception as e: + LOGGER.error("Error populating outputs and labels", error=e) + DB.session.rollback() + + # TODO should this even go here or in its own service? def add_label_to_output( self, txid: str, vout: int, label_display_name: str ) -> list[Label]: """Add a label to an output in the db.""" db_output = OutputModel.query.filter_by(txid=txid, vout=vout).first() label = Label.query.filter_by(display_name=label_display_name).first() + if db_output is None or label is None: + return [] db_output.labels.append(label) DB.session.commit() return db_output.labels @@ -466,7 +558,8 @@ def build_transaction( script, amount_per_recipient_output ) - built_transaction: bdk.TxBuilderResult = tx_builder.finish(self.wallet) + built_transaction: bdk.TxBuilderResult = tx_builder.finish( + self.wallet) built_transaction.transaction_details.transaction return BuildTransactionResponseType( diff --git a/src/app/api/api.ts b/src/app/api/api.ts index 9d94ec05..9823bf21 100644 --- a/src/app/api/api.ts +++ b/src/app/api/api.ts @@ -22,6 +22,9 @@ import { RemoveLabelRequestParams, AddLabelResponseType, RemoveLabelResponseType, + GetOutputLabelsUniqueResponseType, + PopulateOutputLabelsUniqueBodyType, + PopulateOutputLabelsUniqueResponse, } from './types'; import { Network } from '../types/network'; @@ -91,6 +94,29 @@ export class ApiClient { return data as GetOutputLabelsResponseType; } + + static async getOutputLabelsUnique() { + const response = await fetchHandler( + `${configs.backendServerBaseUrl}/transactions/outputs/labels-unique`, + ); + + const data = await response.json(); + + return data as GetOutputLabelsUniqueResponseType; + } + + static async populateOutputLabelsUnique(outputLabels: PopulateOutputLabelsUniqueBodyType) { + const response = await fetchHandler( + `${configs.backendServerBaseUrl}/transactions/outputs/labels-unique`, + 'POST', + outputLabels + ); + + const data = await response.json(); + + return data as PopulateOutputLabelsUniqueResponse; + } + static async addOutputLabel(body: AddLabelRequestBody) { const response = await fetchHandler( `${configs.backendServerBaseUrl}/transactions/outputs/label`, @@ -104,9 +130,9 @@ export class ApiClient { } static async removeOutputLabel( - txid: RemoveLabelRequestParams["txid"], - vout: RemoveLabelRequestParams["vout"], - labelName: RemoveLabelRequestParams["labelName"], + txid: RemoveLabelRequestParams['txid'], + vout: RemoveLabelRequestParams['vout'], + labelName: RemoveLabelRequestParams['labelName'], ) { const response = await fetchHandler( `${configs.backendServerBaseUrl}/transactions/outputs/label?txid=${txid}&vout=${vout}&labelName=${labelName}`, diff --git a/src/app/api/types.ts b/src/app/api/types.ts index 1af1028d..a8650ab5 100644 --- a/src/app/api/types.ts +++ b/src/app/api/types.ts @@ -156,6 +156,16 @@ export type OutputLabelType = { export type GetOutputLabelsResponseType = { labels: [OutputLabelType]; }; +export type GetOutputLabelsUniqueResponseType = { + [key: string]: OutputLabelType; +}; +export type PopulateOutputLabelsUniqueBodyType = { + [key: string]: OutputLabelType; +} + +export type PopulateOutputLabelsUniqueResponse = { + success: boolean; +} export type GetTransactionsResponseType = { txid: string; @@ -194,7 +204,7 @@ export type AddLabelRequestBody = { }; export type AddLabelResponseType = { - labels: [OutputLabelType] + labels: [OutputLabelType]; }; export type RemoveLabelRequestParams = { @@ -204,5 +214,5 @@ export type RemoveLabelRequestParams = { }; export type RemoveLabelResponseType = { - labels: [OutputLabelType] + labels: [OutputLabelType]; }; diff --git a/src/app/components/privacy/txosTable.tsx b/src/app/components/privacy/txosTable.tsx index bfea1c58..05acad84 100644 --- a/src/app/components/privacy/txosTable.tsx +++ b/src/app/components/privacy/txosTable.tsx @@ -18,8 +18,14 @@ import { } from '@tabler/icons-react'; import { BtcMetric, btcSatHandler } from '../../types/btcSatHandler'; import { OutputModal } from '../OutputModal'; -import { TransactionOutputType } from '../../api/types'; -import { useGetOutputLabels } from '../../hooks/transactions'; +import { + GetOutputLabelsUniqueResponseType, + TransactionOutputType, +} from '../../api/types'; +import { + useGetOutputLabels, + useGetOutputLabelsUnique, +} from '../../hooks/transactions'; const sectionColor = 'rgb(1, 67, 97)'; @@ -30,6 +36,16 @@ type TxosTableProps = { export const TxosTable = ({ txos, btcMetric }: TxosTableProps) => { const getOutputLabelsQuery = useGetOutputLabels(); + const saveLabelsInWalletConfig = ( + data: GetOutputLabelsUniqueResponseType, + ) => { + window.electron.ipcRenderer.sendMessage('save-labels', data); + }; + + // get all the output labels in a format that is good for storing + // in the global wallet.labels object via the "save-labels" event + useGetOutputLabelsUnique(saveLabelsInWalletConfig); + const [isOutputModalShowing, setIsOutputModalShowing] = useState(false); const [selectedOutput, setSelectedOutput] = useState(); const columns = useMemo(() => { @@ -142,8 +158,7 @@ export const TxosTable = ({ txos, btcMetric }: TxosTableProps) => { size="sm" className="mt-auto ml-1" > - +
); diff --git a/src/app/hooks/transactions.ts b/src/app/hooks/transactions.ts index 1dd43a1a..91278ffb 100644 --- a/src/app/hooks/transactions.ts +++ b/src/app/hooks/transactions.ts @@ -1,11 +1,17 @@ import { ApiClient } from '../api/api'; -import { useMutation, useQuery } from 'react-query'; -import { AddLabelRequestBody, RemoveLabelRequestParams } from '../api/types'; +import { useMutation, useQuery, useQueryClient } from 'react-query'; +import { + AddLabelRequestBody, + GetOutputLabelsUniqueResponseType, + PopulateOutputLabelsUniqueBodyType, + RemoveLabelRequestParams, +} from '../api/types'; export const uxtoQueryKeys = { getTransactions: ['getTransactions'], getOutputs: ['getOutputs'], getOutputLabels: ['getOutputLabels'], + getOutputLabelsUnique: ['getOutputLabelsUnique'], }; export function useGetTransactions() { @@ -20,7 +26,7 @@ export function useGetTransactions() { export function useGetOutputs() { return useQuery(uxtoQueryKeys.getOutputs, () => ApiClient.getOutputs(), { - refetchOnWindowFocus: true, + refetchOnWindowFocus: false, }); } @@ -29,16 +35,59 @@ export function useGetOutputLabels() { uxtoQueryKeys.getOutputLabels, () => ApiClient.getOutputLabels(), { - refetchOnWindowFocus: false, + refetchOnWindowFocus: true, + }, + ); +} + +export function useGetOutputLabelsUnique( + handleSuccess: (data: GetOutputLabelsUniqueResponseType) => void, +) { + return useQuery( + uxtoQueryKeys.getOutputLabelsUnique, + () => ApiClient.getOutputLabelsUnique(), + { + refetchOnWindowFocus: true, + onSuccess: (data: GetOutputLabelsUniqueResponseType) => { + // save in global wallet storage + // so that when a user saves their wallet the labels are saved + // and then can later repopulate the db on wallet load/import. + handleSuccess(data); + }, + }, + ); +} + +export function usePopulateOutputLabels( + onSuccess?: () => void, + onError?: () => void, +) { + return useMutation( + (body: PopulateOutputLabelsUniqueBodyType) => + ApiClient.populateOutputLabelsUnique(body), + { + onSuccess: () => { + if (onSuccess) { + onSuccess(); + } + }, + onError: () => { + onError(); + }, }, ); } export function useAddUtxoLabel(onError?: () => void) { + const queryClient = useQueryClient(); return useMutation( (addLabelRequestBody: AddLabelRequestBody) => ApiClient.addOutputLabel(addLabelRequestBody), { + onSuccess: () => { + queryClient.invalidateQueries(uxtoQueryKeys.getOutputs); + queryClient.invalidateQueries(uxtoQueryKeys.getOutputLabelsUnique); + }, onError: () => { onError(); }, @@ -47,6 +96,7 @@ export function useAddUtxoLabel(onError?: () => void) { } export function useRemoveUtxoLabel(onError?: () => void) { + const queryClient = useQueryClient(); return useMutation( (requestParams: RemoveLabelRequestParams) => ApiClient.removeOutputLabel( @@ -55,6 +105,14 @@ export function useRemoveUtxoLabel(onError?: () => void) { requestParams.labelName, ), { + onSuccess: () => { + queryClient.invalidateQueries(uxtoQueryKeys.getOutputs); + // we need to invalidate the output labels unique query + // so that the useGetOutputLabelsUnique refires, which will + // populate the global wallet.labels object with the updated labels + // keeping the global wallet.labels object in sync with the latest changes in the db. + queryClient.invalidateQueries(uxtoQueryKeys.getOutputLabelsUnique); + }, onError: () => { onError(); }, diff --git a/src/app/pages/Home.tsx b/src/app/pages/Home.tsx index 364f81cd..abcf5141 100644 --- a/src/app/pages/Home.tsx +++ b/src/app/pages/Home.tsx @@ -27,12 +27,15 @@ import { FeeRateColorChangeInputs } from '../components/FeeRateColorChangeInputs import { CreateTxFeeEstimationResponseType, GetBTCPriceResponseType, + GetOutputLabelsUniqueResponseType, + OutputLabelType, } from '../api/types'; import { Wallet, WalletConfigs } from '../types/wallet'; import { useGetBtcPrice } from '../hooks/price'; import { Pages } from '../../renderer/pages'; import { ScriptTypes } from '../types/scriptTypes'; import { Privacy } from './Privacy'; +import { usePopulateOutputLabels } from '../hooks/transactions'; export type ScaleOption = { value: string; @@ -129,6 +132,9 @@ function Home() { const [feeScale, setFeeScale] = useState(scaleOptions[1]); const [minFeeScale, setMinFeeScale] = useState(minScaleOptions[0]); const [feeRate, setFeeRate] = useState(parseInt(minFeeScale.value)); + // use the labels to populate the backend. + const [importedOutputLabels, setImportedOutputLabels] = + useState(null); // Initially set the current future fee rate to the current medium fee rate // if it was not set by an imported wallet. @@ -202,6 +208,8 @@ function Home() { isCreateBatchTx, ]); + const populateBackendWithLabels = usePopulateOutputLabels(); + const handleWalletData = (walletData?: Wallet) => { console.log('walletData loaded', walletData); const isConfigDataLoaded = @@ -219,9 +227,30 @@ function Home() { setBtcMetric(walletData.btcMetric!); setFeeRateColorMapValues(walletData.feeRateColorMapValues!); } + + const isLabelDataLoaded = !!walletData?.labels; + + if (isLabelDataLoaded) { + setImportedOutputLabels(walletData.labels); + } setHasInitialWalletConfigDataBeenLoaded(true); }; + useEffect(() => { + if ( + populateBackendWithLabels.isLoading || + populateBackendWithLabels.isSuccess + ) { + console.log( + 'The backend has already been populated with labels, therefore do not make the request again.', + ); + } else if (importedOutputLabels && !populateBackendWithLabels.isSuccess) { + populateBackendWithLabels.mutate(importedOutputLabels); + } else{ + console.log("there are no output labels to populate the db with, therefore do not make the request.") + } + }, [importedOutputLabels]); + useEffect(() => { // @ts-ignore window.electron.ipcRenderer.on('wallet-data', handleWalletData); diff --git a/src/app/pages/Privacy.tsx b/src/app/pages/Privacy.tsx index 0b88354c..ceff7545 100644 --- a/src/app/pages/Privacy.tsx +++ b/src/app/pages/Privacy.tsx @@ -15,10 +15,9 @@ export const Privacy = ({ btcMetric }: PrivacyProps) => { PREVIEW = 'preview', } - const transactionsResponse = useGetTransactions(); - console.log('transactionsResponse', transactionsResponse.data); + // TODO add back in and use when using the transaction tab + // const transactionsResponse = useGetTransactions(); const outputs = useGetOutputs(); - console.log('outputs', outputs.data); return (
{ feeScale: importedFeeScale, minFeeScale: importedMinFeeScale, feeRate: importedFeeRate, + labels: importedLabels, } = walletData; console.log('imported descriptor', importedDefaultDescriptor); @@ -530,6 +531,7 @@ export const WalletSignIn = () => { minFeeScale: importedMinFeeScale, feeRate: importedFeeRate, isCreateBatchTx: importedIsCreateBatchTx, + labels: importedLabels, }; window.electron.ipcRenderer.sendMessage( 'save-wallet-configs', diff --git a/src/app/types/wallet.ts b/src/app/types/wallet.ts index 486fa764..c10ebc90 100644 --- a/src/app/types/wallet.ts +++ b/src/app/types/wallet.ts @@ -1,3 +1,4 @@ +import { GetOutputLabelsResponseType, OutputLabelType } from '../api/types'; import { PolicyTypeOption } from '../components/formOptions'; import { FeeRateColor, ScaleOption } from '../pages/Home'; import { BtcMetric } from './btcSatHandler'; @@ -24,6 +25,7 @@ export type Wallet = { feeScale?: ScaleOption; minFeeScale?: ScaleOption; feeRate?: string | number; + labels?: GetOutputLabelsResponseType; }; export type WalletConfigs = { diff --git a/src/main/main.ts b/src/main/main.ts index 5ca9960b..4edfbba9 100644 --- a/src/main/main.ts +++ b/src/main/main.ts @@ -21,6 +21,7 @@ import { } from './util'; import { WalletConfigs } from '../app/types/wallet'; import { Pages } from '../renderer/pages'; +import { OutputLabelType } from '../app/api/types'; class AppUpdater { constructor() { @@ -67,6 +68,11 @@ ipcMain.on('current-route', (event, currentRoute) => { } }); +ipcMain.on('save-labels', async (event, labels: [OutputLabelType]) => { + const menu = MenuBuilder.menu; + menu.walletDetails.labels = labels; +}); + ipcMain.on('save-wallet', async (event, walletDetails) => { const menu = MenuBuilder.menu; menu.walletDetails = walletDetails; diff --git a/src/main/preload.ts b/src/main/preload.ts index aa4791bf..1fb5885f 100644 --- a/src/main/preload.ts +++ b/src/main/preload.ts @@ -13,7 +13,8 @@ export type Channels = | 'save-wallet-data-from-dialog' | 'save-psbt' | 'logout' - | 'wallet-data'; + | 'wallet-data' + | 'save-labels'; const electronHandler = { ipcRenderer: { From 4873ba180d8ca28a075f242bbc2ddffa35d28ed1 Mon Sep 17 00:00:00 2001 From: Joseph Wyman Date: Sun, 27 Oct 2024 08:07:34 -0400 Subject: [PATCH 15/85] remove bad unique name and replace it with populate name --- backend/src/controllers/transactions.py | 33 +++++++++---------- backend/src/my_types/__init__.py | 6 ++-- .../my_types/controller_types/utxos_dtos.py | 6 ++-- backend/src/services/wallet/wallet.py | 6 ++-- src/app/api/api.ts | 20 ++++++----- src/app/api/types.ts | 10 +++--- src/app/components/privacy/txosTable.tsx | 4 +-- src/app/hooks/transactions.ts | 10 +++--- src/app/pages/Home.tsx | 14 ++++---- 9 files changed, 54 insertions(+), 55 deletions(-) diff --git a/backend/src/controllers/transactions.py b/backend/src/controllers/transactions.py index 94c7d091..fc4c2907 100644 --- a/backend/src/controllers/transactions.py +++ b/backend/src/controllers/transactions.py @@ -14,14 +14,13 @@ RemoveOutputLabelResponseDto, AddOutputLabelResponseDto, GetOutputLabelsResponseDto, - GetOutputLabelsUniqueResponseDto, - PopulateOutputLabelsUniqueResponseDto, - PopulateOutputLabelsUniqueRequestDto, + GetOutputLabelsPopulateResponseDto, + PopulateOutputLabelsResponseDto, + PopulateOutputLabelsRequestDto, ) from src.my_types.controller_types.generic_response_types import SimpleErrorResponse -transactions_page = Blueprint( - "get_transactions", __name__, url_prefix="/transactions") +transactions_page = Blueprint("get_transactions", __name__, url_prefix="/transactions") LOGGER = structlog.get_logger() @@ -38,8 +37,7 @@ def get_txos( transactions = wallet_service.get_all_transactions() return GetAllTransactionsResponseDto.model_validate( - dict(transactions=[transaction.as_dict() - for transaction in transactions]) + dict(transactions=[transaction.as_dict() for transaction in transactions]) ).model_dump() except Exception as e: @@ -89,30 +87,29 @@ def get_output_labels( ).model_dump() -# TODO better name -@transactions_page.route("/outputs/labels-unique", methods=["GET"]) +@transactions_page.route("/outputs/populate-labels", methods=["GET"]) @inject def get_output_labels_unique( wallet_service: WalletService = Provide[ServiceContainer.wallet_service], ): """ - Get all labels for each txid-vout + Get all labels for each txid-vout, used for being saved locally and + eventually repopulating the database. """ try: labels = wallet_service.get_output_labels_unique() - return GetOutputLabelsUniqueResponseDto.model_validate(labels).model_dump() + return GetOutputLabelsPopulateResponseDto.model_validate(labels).model_dump() except Exception as e: - LOGGER.error("error getting all output labels unique", error=e) + LOGGER.error("error getting all populate output labels", error=e) return SimpleErrorResponse( - message="error getting all output labels unique" + message="error getting all populate output labels" ).model_dump() -# TODO better name for endpoint -@transactions_page.route("/outputs/labels-unique", methods=["POST"]) +@transactions_page.route("/outputs/populate-labels", methods=["POST"]) @inject -def populate_output_labels_unique( +def populate_output_labels( wallet_service: WalletService = Provide[ServiceContainer.wallet_service], ): """ @@ -122,10 +119,10 @@ def populate_output_labels_unique( for their wallet's outputs that they have saved. """ try: - data = PopulateOutputLabelsUniqueRequestDto(json.loads(request.data)) + data = PopulateOutputLabelsRequestDto(json.loads(request.data)) wallet_service.populate_outputs_and_labels(data) - return PopulateOutputLabelsUniqueResponseDto.model_validate( + return PopulateOutputLabelsResponseDto.model_validate( {"success": True} ).model_dump() # diff --git a/backend/src/my_types/__init__.py b/backend/src/my_types/__init__.py index eb0535b4..0b18b1cf 100644 --- a/backend/src/my_types/__init__.py +++ b/backend/src/my_types/__init__.py @@ -13,9 +13,9 @@ AddOutputLabelResponseDto, RemoveOutputLabelResponseDto, GetOutputLabelsResponseDto, - GetOutputLabelsUniqueResponseDto, - PopulateOutputLabelsUniqueResponseDto, - PopulateOutputLabelsUniqueRequestDto, + GetOutputLabelsPopulateResponseDto, + PopulateOutputLabelsResponseDto, + PopulateOutputLabelsRequestDto, ) from src.my_types.controller_types.fees_dtos import GetCurrentFeesResponseDto diff --git a/backend/src/my_types/controller_types/utxos_dtos.py b/backend/src/my_types/controller_types/utxos_dtos.py index 698cf9ad..53bc62e2 100644 --- a/backend/src/my_types/controller_types/utxos_dtos.py +++ b/backend/src/my_types/controller_types/utxos_dtos.py @@ -160,13 +160,13 @@ class GetOutputLabelsResponseDto(BaseModel): labels: list[OutputLabelDto] -class GetOutputLabelsUniqueResponseDto(RootModel): +class GetOutputLabelsPopulateResponseDto(RootModel): root: Dict[str, List[OutputLabelDto]] -class PopulateOutputLabelsUniqueRequestDto(RootModel): +class PopulateOutputLabelsRequestDto(RootModel): root: Dict[str, List[OutputLabelDto]] -class PopulateOutputLabelsUniqueResponseDto(BaseModel): +class PopulateOutputLabelsResponseDto(BaseModel): success: bool diff --git a/backend/src/services/wallet/wallet.py b/backend/src/services/wallet/wallet.py index 516f0ca7..af5e21f9 100644 --- a/backend/src/services/wallet/wallet.py +++ b/backend/src/services/wallet/wallet.py @@ -20,7 +20,7 @@ ) from src.my_types.controller_types.utxos_dtos import ( OutputLabelDto, - PopulateOutputLabelsUniqueRequestDto, + PopulateOutputLabelsRequestDto, ) from src.my_types.transactions import LiveWalletOutput from src.services.wallet.raw_output_script_examples import ( @@ -455,10 +455,10 @@ def get_output_labels_unique( return result def populate_outputs_and_labels( - self, unique_output_labels: PopulateOutputLabelsUniqueRequestDto + self, populate_output_labels: PopulateOutputLabelsRequestDto ) -> None: # TODO maybe a success of fail reutn type? try: - model_dump = unique_output_labels.model_dump() + model_dump = populate_output_labels.model_dump() for unique_output_txid_vout in model_dump.keys(): txid, vout = unique_output_txid_vout.split("-") # db_output = OutputModel.query.filter_by(txid=txid, vout=vout).first() diff --git a/src/app/api/api.ts b/src/app/api/api.ts index 9823bf21..61492431 100644 --- a/src/app/api/api.ts +++ b/src/app/api/api.ts @@ -22,9 +22,9 @@ import { RemoveLabelRequestParams, AddLabelResponseType, RemoveLabelResponseType, - GetOutputLabelsUniqueResponseType, - PopulateOutputLabelsUniqueBodyType, - PopulateOutputLabelsUniqueResponse, + GetOutputLabelsPopulateResponseType, + PopulateOutputLabelsBodyType, + PopulateOutputLabelsResponse, } from './types'; import { Network } from '../types/network'; @@ -97,24 +97,26 @@ export class ApiClient { static async getOutputLabelsUnique() { const response = await fetchHandler( - `${configs.backendServerBaseUrl}/transactions/outputs/labels-unique`, + `${configs.backendServerBaseUrl}/transactions/outputs/populate-labels`, ); const data = await response.json(); - return data as GetOutputLabelsUniqueResponseType; + return data as GetOutputLabelsPopulateResponseType; } - static async populateOutputLabelsUnique(outputLabels: PopulateOutputLabelsUniqueBodyType) { + static async populateOutputLabelsUnique( + outputLabels: PopulateOutputLabelsBodyType, + ) { const response = await fetchHandler( - `${configs.backendServerBaseUrl}/transactions/outputs/labels-unique`, + `${configs.backendServerBaseUrl}/transactions/outputs/populate-labels`, 'POST', - outputLabels + outputLabels, ); const data = await response.json(); - return data as PopulateOutputLabelsUniqueResponse; + return data as PopulateOutputLabelsResponse; } static async addOutputLabel(body: AddLabelRequestBody) { diff --git a/src/app/api/types.ts b/src/app/api/types.ts index a8650ab5..d93cd586 100644 --- a/src/app/api/types.ts +++ b/src/app/api/types.ts @@ -156,16 +156,16 @@ export type OutputLabelType = { export type GetOutputLabelsResponseType = { labels: [OutputLabelType]; }; -export type GetOutputLabelsUniqueResponseType = { +export type GetOutputLabelsPopulateResponseType = { [key: string]: OutputLabelType; }; -export type PopulateOutputLabelsUniqueBodyType = { +export type PopulateOutputLabelsBodyType = { [key: string]: OutputLabelType; -} +}; -export type PopulateOutputLabelsUniqueResponse = { +export type PopulateOutputLabelsResponse = { success: boolean; -} +}; export type GetTransactionsResponseType = { txid: string; diff --git a/src/app/components/privacy/txosTable.tsx b/src/app/components/privacy/txosTable.tsx index 05acad84..aebd366a 100644 --- a/src/app/components/privacy/txosTable.tsx +++ b/src/app/components/privacy/txosTable.tsx @@ -19,7 +19,7 @@ import { import { BtcMetric, btcSatHandler } from '../../types/btcSatHandler'; import { OutputModal } from '../OutputModal'; import { - GetOutputLabelsUniqueResponseType, + GetOutputLabelsPopulateResponseType, TransactionOutputType, } from '../../api/types'; import { @@ -37,7 +37,7 @@ type TxosTableProps = { export const TxosTable = ({ txos, btcMetric }: TxosTableProps) => { const getOutputLabelsQuery = useGetOutputLabels(); const saveLabelsInWalletConfig = ( - data: GetOutputLabelsUniqueResponseType, + data: GetOutputLabelsPopulateResponseType, ) => { window.electron.ipcRenderer.sendMessage('save-labels', data); }; diff --git a/src/app/hooks/transactions.ts b/src/app/hooks/transactions.ts index 91278ffb..6b125d2c 100644 --- a/src/app/hooks/transactions.ts +++ b/src/app/hooks/transactions.ts @@ -2,8 +2,8 @@ import { ApiClient } from '../api/api'; import { useMutation, useQuery, useQueryClient } from 'react-query'; import { AddLabelRequestBody, - GetOutputLabelsUniqueResponseType, - PopulateOutputLabelsUniqueBodyType, + GetOutputLabelsPopulateResponseType, + PopulateOutputLabelsBodyType, RemoveLabelRequestParams, } from '../api/types'; @@ -41,14 +41,14 @@ export function useGetOutputLabels() { } export function useGetOutputLabelsUnique( - handleSuccess: (data: GetOutputLabelsUniqueResponseType) => void, + handleSuccess: (data: GetOutputLabelsPopulateResponseType) => void, ) { return useQuery( uxtoQueryKeys.getOutputLabelsUnique, () => ApiClient.getOutputLabelsUnique(), { refetchOnWindowFocus: true, - onSuccess: (data: GetOutputLabelsUniqueResponseType) => { + onSuccess: (data: GetOutputLabelsPopulateResponseType) => { // save in global wallet storage // so that when a user saves their wallet the labels are saved // and then can later repopulate the db on wallet load/import. @@ -63,7 +63,7 @@ export function usePopulateOutputLabels( onError?: () => void, ) { return useMutation( - (body: PopulateOutputLabelsUniqueBodyType) => + (body: PopulateOutputLabelsBodyType) => ApiClient.populateOutputLabelsUnique(body), { onSuccess: () => { diff --git a/src/app/pages/Home.tsx b/src/app/pages/Home.tsx index abcf5141..192e1814 100644 --- a/src/app/pages/Home.tsx +++ b/src/app/pages/Home.tsx @@ -15,20 +15,18 @@ import { SegmentedControl, ActionIcon, NumberInput, - Tooltip, Collapse, } from '@mantine/core'; import { useDeleteCurrentWallet, useGetWalletType } from '../hooks/wallet'; import { useQueryClient } from 'react-query'; import { BtcMetric, btcSatHandler } from '../types/btcSatHandler'; import { SettingsSlideout } from '../components/SettingsSlideout'; -import { IconAdjustments, IconInfoCircle } from '@tabler/icons-react'; +import { IconAdjustments } from '@tabler/icons-react'; import { FeeRateColorChangeInputs } from '../components/FeeRateColorChangeInputs'; import { CreateTxFeeEstimationResponseType, GetBTCPriceResponseType, - GetOutputLabelsUniqueResponseType, - OutputLabelType, + GetOutputLabelsPopulateResponseType, } from '../api/types'; import { Wallet, WalletConfigs } from '../types/wallet'; import { useGetBtcPrice } from '../hooks/price'; @@ -134,7 +132,7 @@ function Home() { const [feeRate, setFeeRate] = useState(parseInt(minFeeScale.value)); // use the labels to populate the backend. const [importedOutputLabels, setImportedOutputLabels] = - useState(null); + useState(null); // Initially set the current future fee rate to the current medium fee rate // if it was not set by an imported wallet. @@ -246,8 +244,10 @@ function Home() { ); } else if (importedOutputLabels && !populateBackendWithLabels.isSuccess) { populateBackendWithLabels.mutate(importedOutputLabels); - } else{ - console.log("there are no output labels to populate the db with, therefore do not make the request.") + } else { + console.log( + 'there are no output labels to populate the db with, therefore do not make the request.', + ); } }, [importedOutputLabels]); From d25280ba29868ecdd2186287cdf90b4a9ac1bdb9 Mon Sep 17 00:00:00 2001 From: Joseph Wyman Date: Sun, 27 Oct 2024 10:49:55 -0400 Subject: [PATCH 16/85] write label related controller tests --- backend/src/controllers/transactions.py | 93 ++++----- .../my_types/controller_types/utxos_dtos.py | 2 +- .../test_transactions_controller.py | 187 +++++++++++++++++- 3 files changed, 230 insertions(+), 52 deletions(-) diff --git a/backend/src/controllers/transactions.py b/backend/src/controllers/transactions.py index fc4c2907..ac22125b 100644 --- a/backend/src/controllers/transactions.py +++ b/backend/src/controllers/transactions.py @@ -87,52 +87,6 @@ def get_output_labels( ).model_dump() -@transactions_page.route("/outputs/populate-labels", methods=["GET"]) -@inject -def get_output_labels_unique( - wallet_service: WalletService = Provide[ServiceContainer.wallet_service], -): - """ - Get all labels for each txid-vout, used for being saved locally and - eventually repopulating the database. - """ - try: - labels = wallet_service.get_output_labels_unique() - - return GetOutputLabelsPopulateResponseDto.model_validate(labels).model_dump() - except Exception as e: - LOGGER.error("error getting all populate output labels", error=e) - return SimpleErrorResponse( - message="error getting all populate output labels" - ).model_dump() - - -@transactions_page.route("/outputs/populate-labels", methods=["POST"]) -@inject -def populate_output_labels( - wallet_service: WalletService = Provide[ServiceContainer.wallet_service], -): - """ - Populate all labels and outputs for each txid-vout - - This is used when a user loads a wallet and needs to populate the labels - for their wallet's outputs that they have saved. - """ - try: - data = PopulateOutputLabelsRequestDto(json.loads(request.data)) - wallet_service.populate_outputs_and_labels(data) - - return PopulateOutputLabelsResponseDto.model_validate( - {"success": True} - ).model_dump() - # - except Exception as e: - LOGGER.error("error populating all labels and outputs.", error=e) - return SimpleErrorResponse( - message="error populating all labels and outputs." - ).model_dump() - - @transactions_page.route("/outputs/label", methods=["POST"]) @inject def add_output_label( @@ -145,6 +99,7 @@ def add_output_label( add_output_label_request_dto = AddOutputLabelRequestDto.model_validate( json.loads(request.data) ) + labels = wallet_service.add_label_to_output( add_output_label_request_dto.txid, add_output_label_request_dto.vout, @@ -211,3 +166,49 @@ def remove_output_label( return SimpleErrorResponse( message="Error removing a label from an output." ).model_dump() + + +@transactions_page.route("/outputs/populate-labels", methods=["GET"]) +@inject +def get_output_labels_unique( + wallet_service: WalletService = Provide[ServiceContainer.wallet_service], +): + """ + Get all labels for each txid-vout, used for being saved locally and + eventually repopulating the database. + """ + try: + labels = wallet_service.get_output_labels_unique() + + return GetOutputLabelsPopulateResponseDto.model_validate(labels).model_dump() + except Exception as e: + LOGGER.error("error getting all populate output labels", error=e) + return SimpleErrorResponse( + message="error getting all populate output labels" + ).model_dump() + + +@transactions_page.route("/outputs/populate-labels", methods=["POST"]) +@inject +def populate_output_labels( + wallet_service: WalletService = Provide[ServiceContainer.wallet_service], +): + """ + Populate all labels and outputs for each txid-vout + + This is used when a user loads a wallet and needs to populate the labels + for their wallet's outputs that they have saved. + """ + try: + data = PopulateOutputLabelsRequestDto(json.loads(request.data)) + wallet_service.populate_outputs_and_labels(data) + + return PopulateOutputLabelsResponseDto.model_validate( + {"success": True} + ).model_dump() + # + except Exception as e: + LOGGER.error("error populating all labels and outputs.", error=e) + return SimpleErrorResponse( + message="error populating all labels and outputs." + ).model_dump() diff --git a/backend/src/my_types/controller_types/utxos_dtos.py b/backend/src/my_types/controller_types/utxos_dtos.py index 53bc62e2..a8d0539b 100644 --- a/backend/src/my_types/controller_types/utxos_dtos.py +++ b/backend/src/my_types/controller_types/utxos_dtos.py @@ -133,7 +133,7 @@ class GetAllOutputsResponseDto(BaseModel): class AddOutputLabelRequestDto(BaseModel): txid: str vout: int - labelName: str + labelName: str # TODO change to displayName class RemoveOutputLabelRequestDto(BaseModel): diff --git a/backend/src/tests/controller_tests/test_transactions_controller.py b/backend/src/tests/controller_tests/test_transactions_controller.py index 52a8d90d..c3571c15 100644 --- a/backend/src/tests/controller_tests/test_transactions_controller.py +++ b/backend/src/tests/controller_tests/test_transactions_controller.py @@ -1,8 +1,13 @@ from unittest import TestCase -from unittest.mock import MagicMock +from unittest.mock import MagicMock, Mock from typing import List from src.app import AppCreator +from src.models.label import Label +from src.my_types.controller_types.utxos_dtos import ( + OutputLabelDto, + PopulateOutputLabelsRequestDto, +) from src.services.wallet.wallet import WalletService from src.tests.mocks import all_transactions_mock import json @@ -20,7 +25,8 @@ def setUp(self): ) def test_get_transactions(self): - get_all_transactions_mock = MagicMock(return_value=all_transactions_mock) + get_all_transactions_mock = MagicMock( + return_value=all_transactions_mock) with self.app.container.wallet_service.override(self.mock_wallet_service): self.mock_wallet_service.get_all_transactions = get_all_transactions_mock get_transactions_response = self.test_client.get("/transactions/") @@ -33,12 +39,15 @@ def test_get_transactions(self): } def test_get_utxos(self): - output_lists: List[List[Output]] = [tx.outputs for tx in all_transactions_mock] - all_outputs = [output for output_list in output_lists for output in output_list] + output_lists: List[List[Output]] = [ + tx.outputs for tx in all_transactions_mock] + all_outputs = [ + output for output_list in output_lists for output in output_list] get_all_outputs_mock = MagicMock(return_value=all_outputs) with self.app.container.wallet_service.override(self.mock_wallet_service): self.mock_wallet_service.get_all_outputs = get_all_outputs_mock - get_all_outputs_response = self.test_client.get("transactions/outputs") + get_all_outputs_response = self.test_client.get( + "transactions/outputs") get_all_outputs_mock.assert_called_once() @@ -46,3 +55,171 @@ def test_get_utxos(self): assert json.loads(get_all_outputs_response.data) == { "outputs": [output.as_dict() for output in all_outputs] } + + def test_get_output_labels(self): + output_labels = [ + OutputLabelDto( + label="mock_label", + display_name="mock_display_name", + description="mock_display_description", + ) + ] + get_all_output_labels = MagicMock(return_value=output_labels) + with self.app.container.wallet_service.override(self.mock_wallet_service): + self.mock_wallet_service.get_output_labels = get_all_output_labels + get_all_output_labels_response = self.test_client.get( + "transactions/outputs/labels" + ) + + get_all_output_labels.assert_called_once() + + assert get_all_output_labels_response.status == "200 OK" + assert json.loads(get_all_output_labels_response.data) == { + "labels": [label.model_dump() for label in output_labels] + } + + def test_post_output_labels(self): + output_label_mock_one = Mock() + output_label_mock_one.name = "mock_name_1" + output_label_mock_one.display_name = "mock_display_name_1" + output_label_mock_one.description = "mock_description_1" + + output_label_mock_two = Mock() + output_label_mock_two.name = "mock_name_2" + output_label_mock_two.display_name = "mock_display_name_2" + output_label_mock_two.description = "mock_description_2" + + output_labels = [output_label_mock_one, output_label_mock_two] + add_label_to_output_mock = MagicMock(return_value=output_labels) + output_label_request_body = { + "txid": "mockTxId", + "vout": 0, + "labelName": "mockLabelName", + } + with self.app.container.wallet_service.override(self.mock_wallet_service): + self.mock_wallet_service.add_label_to_output = add_label_to_output_mock + post_output_labels_response = self.test_client.post( + "transactions/outputs/label", + json=output_label_request_body, + ) + + add_label_to_output_mock.assert_called_once_with( + "mockTxId", 0, "mockLabelName" + ) + + assert post_output_labels_response.status == "200 OK" + assert json.loads(post_output_labels_response.data) == { + "labels": [ + dict( + label=label.name, + display_name=label.display_name, + description=label.description, + ) + for label in output_labels + ] + } + + def test_remove_output_label(self): + output_label_mock_one = Mock() + output_label_mock_one.name = "mock_name_1" + output_label_mock_one.display_name = "mock_display_name_1" + output_label_mock_one.description = "mock_description_1" + + output_labels = [output_label_mock_one] + remove_label_from_output_mock = MagicMock(return_value=output_labels) + txid_request_param = "mockTxId" + vout_request_param = 0 + labelName_request_param = "labelNameMock" + with self.app.container.wallet_service.override(self.mock_wallet_service): + self.mock_wallet_service.remove_label_from_output = ( + remove_label_from_output_mock + ) + + remove_output_labels_response = self.test_client.delete( + f"transactions/outputs/label?txid={txid_request_param}&vout={vout_request_param}&labelName={labelName_request_param}", + ) + + remove_label_from_output_mock.assert_called_once_with( + txid_request_param, vout_request_param, labelName_request_param + ) + + assert remove_output_labels_response.status == "200 OK" + + # should return existing labels + assert json.loads(remove_output_labels_response.data) == { + "labels": [ + dict( + label=label.name, + display_name=label.display_name, + description=label.description, + ) + for label in output_labels + ] + } + + def test_get_populate_labels(self): + output_label_mock_one = OutputLabelDto( + label="mock_label", + display_name="mock_display_name", + description="mock_display_description", + ) + + mock_populate_labels = {"mock-txid-0": [output_label_mock_one]} + get_output_labels_unique_mock = MagicMock( + return_value=mock_populate_labels) + + with self.app.container.wallet_service.override(self.mock_wallet_service): + self.mock_wallet_service.get_output_labels_unique = ( + get_output_labels_unique_mock + ) + + get_output_populate_labels_response = self.test_client.get( + f"transactions/outputs/populate-labels", + ) + + get_output_labels_unique_mock.assert_called_once() + + assert get_output_populate_labels_response.status == "200 OK" + + # should return existing labels + assert json.loads(get_output_populate_labels_response.data) == { + "mock-txid-0": [ + { + "label": "mock_label", + "display_name": "mock_display_name", + "description": "mock_display_description", + } + ] + } + + def test_post_populate_labels(self): + populate_outputs_and_labels_mock = MagicMock(return_value=None) + + mock_populate_labels_body = { + "mock-txid-0": [ + { + "label": "mock_label", + "display_name": "mock_display_name", + "description": "mock_display_description", + } + ] + } + + with self.app.container.wallet_service.override(self.mock_wallet_service): + self.mock_wallet_service.populate_outputs_and_labels = ( + populate_outputs_and_labels_mock + ) + + get_output_populate_labels_response = self.test_client.post( + f"transactions/outputs/populate-labels", json=mock_populate_labels_body + ) + + populate_outputs_and_labels_mock.assert_called_once_with( + PopulateOutputLabelsRequestDto(mock_populate_labels_body) + ) + + assert get_output_populate_labels_response.status == "200 OK" + + assert json.loads(get_output_populate_labels_response.data) == { + "success": True + } From 23f25239937eeb6a97f622a4e3be528985d326df Mon Sep 17 00:00:00 2001 From: Joseph Wyman Date: Sun, 27 Oct 2024 12:36:52 -0400 Subject: [PATCH 17/85] fix all broken backend tests due to latest label related changes --- backend/src/services/wallet/wallet.py | 15 ++--- .../test_transactions_controller.py | 17 ++---- .../test_wallet_controller.py | 37 +++++------ backend/src/tests/mocks.py | 13 +++- .../service_tests/test_wallet_service.py | 61 ++++++++++++++++--- 5 files changed, 90 insertions(+), 53 deletions(-) diff --git a/backend/src/services/wallet/wallet.py b/backend/src/services/wallet/wallet.py index af5e21f9..4c7a5426 100644 --- a/backend/src/services/wallet/wallet.py +++ b/backend/src/services/wallet/wallet.py @@ -161,8 +161,7 @@ def connect_wallet( ) wallet_change_descriptor = ( - bdk.Descriptor(change_descriptor, - bdk.Network._value2member_map_[network]) + bdk.Descriptor(change_descriptor, bdk.Network._value2member_map_[network]) if change_descriptor else None ) @@ -184,8 +183,7 @@ def connect_wallet( database_config=db_config, ) - LOGGER.info( - f"Connecting a new wallet to electrum server {wallet_details_id}") + LOGGER.info(f"Connecting a new wallet to electrum server {wallet_details_id}") LOGGER.info(f"xpub {wallet_descriptor.as_string()}") wallet.sync(blockchain, None) @@ -234,8 +232,7 @@ def create_spendable_descriptor( twelve_word_secret = bdk.Mnemonic(bdk.WordCount.WORDS12) # xpriv - descriptor_secret_key = bdk.DescriptorSecretKey( - network, twelve_word_secret, "") + descriptor_secret_key = bdk.DescriptorSecretKey(network, twelve_word_secret, "") wallet_descriptor = None if script_type == ScriptType.P2PKH: @@ -342,8 +339,7 @@ def get_all_outputs(self) -> List[LiveWalletOutput]: all_transactions = self.get_all_transactions() all_outputs: List[LiveWalletOutput] = [] for transaction in all_transactions: - annominity_sets = self.calculate_output_annominity_sets( - transaction.outputs) + annominity_sets = self.calculate_output_annominity_sets(transaction.outputs) for output in transaction.outputs: db_output = self.sync_local_db_with_incoming_output( txid=transaction.txid, vout=output.output_n @@ -558,8 +554,7 @@ def build_transaction( script, amount_per_recipient_output ) - built_transaction: bdk.TxBuilderResult = tx_builder.finish( - self.wallet) + built_transaction: bdk.TxBuilderResult = tx_builder.finish(self.wallet) built_transaction.transaction_details.transaction return BuildTransactionResponseType( diff --git a/backend/src/tests/controller_tests/test_transactions_controller.py b/backend/src/tests/controller_tests/test_transactions_controller.py index c3571c15..9c2a7daa 100644 --- a/backend/src/tests/controller_tests/test_transactions_controller.py +++ b/backend/src/tests/controller_tests/test_transactions_controller.py @@ -8,8 +8,9 @@ OutputLabelDto, PopulateOutputLabelsRequestDto, ) +from src.my_types.transactions import LiveWalletOutput from src.services.wallet.wallet import WalletService -from src.tests.mocks import all_transactions_mock +from src.tests.mocks import all_transactions_mock, all_outputs_mock import json from bitcoinlib.transactions import Output @@ -25,8 +26,7 @@ def setUp(self): ) def test_get_transactions(self): - get_all_transactions_mock = MagicMock( - return_value=all_transactions_mock) + get_all_transactions_mock = MagicMock(return_value=all_transactions_mock) with self.app.container.wallet_service.override(self.mock_wallet_service): self.mock_wallet_service.get_all_transactions = get_all_transactions_mock get_transactions_response = self.test_client.get("/transactions/") @@ -39,15 +39,11 @@ def test_get_transactions(self): } def test_get_utxos(self): - output_lists: List[List[Output]] = [ - tx.outputs for tx in all_transactions_mock] - all_outputs = [ - output for output_list in output_lists for output in output_list] + all_outputs = all_outputs_mock get_all_outputs_mock = MagicMock(return_value=all_outputs) with self.app.container.wallet_service.override(self.mock_wallet_service): self.mock_wallet_service.get_all_outputs = get_all_outputs_mock - get_all_outputs_response = self.test_client.get( - "transactions/outputs") + get_all_outputs_response = self.test_client.get("transactions/outputs") get_all_outputs_mock.assert_called_once() @@ -165,8 +161,7 @@ def test_get_populate_labels(self): ) mock_populate_labels = {"mock-txid-0": [output_label_mock_one]} - get_output_labels_unique_mock = MagicMock( - return_value=mock_populate_labels) + get_output_labels_unique_mock = MagicMock(return_value=mock_populate_labels) with self.app.container.wallet_service.override(self.mock_wallet_service): self.mock_wallet_service.get_output_labels_unique = ( diff --git a/backend/src/tests/controller_tests/test_wallet_controller.py b/backend/src/tests/controller_tests/test_wallet_controller.py index 7e2194fa..d8357bc6 100644 --- a/backend/src/tests/controller_tests/test_wallet_controller.py +++ b/backend/src/tests/controller_tests/test_wallet_controller.py @@ -134,30 +134,30 @@ def test_get_wallet_type_error(self): def test_remove_wallet_success(self): with patch("src.controllers.wallet.WalletService") as wallet_service_mock: wallet_service_mock.remove_global_wallet_and_details = MagicMock() + wallet_service_mock.remove_output_and_related_label_data = MagicMock() wallet_response = self.test_client.delete( "/wallet/remove", ) wallet_service_mock.remove_global_wallet_and_details.assert_called_once() + wallet_service_mock.remove_output_and_related_label_data.assert_called_once() assert wallet_response.status == "200 OK" assert json.loads(wallet_response.data) == { - "message": "wallet successfully deleted", + "message": "wallet and related data successfully deleted", } def test_spendable_success(self): self.mock_wallet_service = MagicMock(WalletService) spendable_descriptor_mock = MagicMock(bdk.Descriptor) - spendable_descriptor_mock.as_string = MagicMock( - return_value="mock_descriptor") + spendable_descriptor_mock.as_string = MagicMock(return_value="mock_descriptor") self.mock_wallet_service.create_spendable_descriptor = MagicMock( return_value=spendable_descriptor_mock ) wallet_mock = MagicMock(bdk.Wallet) get_address_mock = MagicMock() address_mock = MagicMock(return_value=get_address_mock) - get_address_mock.address.as_string = MagicMock( - return_value="mock_address") + get_address_mock.address.as_string = MagicMock(return_value="mock_address") wallet_mock.get_address = address_mock self.mock_wallet_service.create_spendable_wallet = MagicMock( return_value=wallet_mock @@ -210,16 +210,14 @@ def test_spendable__with_descriptor_param_success(self): self.mock_wallet_service = MagicMock(WalletService) spendable_descriptor_mock = MagicMock(bdk.Descriptor) - spendable_descriptor_mock.as_string = MagicMock( - return_value="mock_descriptor") + spendable_descriptor_mock.as_string = MagicMock(return_value="mock_descriptor") self.mock_wallet_service.create_spendable_descriptor = MagicMock( return_value=spendable_descriptor_mock ) wallet_mock = MagicMock(bdk.Wallet) get_address_mock = MagicMock() address_mock = MagicMock(return_value=get_address_mock) - get_address_mock.address.as_string = MagicMock( - return_value="mock_address") + get_address_mock.address.as_string = MagicMock(return_value="mock_address") wallet_mock.get_address = address_mock self.mock_wallet_service.create_spendable_wallet = MagicMock( return_value=wallet_mock @@ -281,16 +279,14 @@ def test_spendable_descriptor_error(self): self.mock_wallet_service = MagicMock(WalletService) spendable_descriptor_mock = MagicMock(bdk.Descriptor) - spendable_descriptor_mock.as_string = MagicMock( - return_value="mock_descriptor") + spendable_descriptor_mock.as_string = MagicMock(return_value="mock_descriptor") self.mock_wallet_service.create_spendable_descriptor = MagicMock( return_value=spendable_descriptor_mock ) wallet_mock = MagicMock(bdk.Wallet) get_address_mock = MagicMock() address_mock = MagicMock(return_value=get_address_mock) - get_address_mock.address.as_string = MagicMock( - return_value="mock_address") + get_address_mock.address.as_string = MagicMock(return_value="mock_address") wallet_mock.get_address = address_mock self.mock_wallet_service.create_spendable_wallet = MagicMock( return_value=wallet_mock @@ -335,16 +331,14 @@ def test_spendable_randomly_fund_mock_wallet_error(self): self.mock_wallet_service = MagicMock(WalletService) spendable_descriptor_mock = MagicMock(bdk.Descriptor) - spendable_descriptor_mock.as_string = MagicMock( - return_value="mock_descriptor") + spendable_descriptor_mock.as_string = MagicMock(return_value="mock_descriptor") self.mock_wallet_service.create_spendable_descriptor = MagicMock( return_value=spendable_descriptor_mock ) wallet_mock = MagicMock(bdk.Wallet) get_address_mock = MagicMock() address_mock = MagicMock(return_value=get_address_mock) - get_address_mock.address.as_string = MagicMock( - return_value="mock_address") + get_address_mock.address.as_string = MagicMock(return_value="mock_address") wallet_mock.get_address = address_mock self.mock_wallet_service.create_spendable_wallet = MagicMock( return_value=wallet_mock @@ -364,8 +358,7 @@ def test_spendable_randomly_fund_mock_wallet_error(self): WalletService, "create_spendable_wallet", return_value=wallet_mock ) as create_spendable_wallet_mock, ): - randomly_fund_mock_wallet_mock.side_effect = Exception( - "mock exception") + randomly_fund_mock_wallet_mock.side_effect = Exception("mock exception") wallet_response = self.test_client.post( "/wallet/spendable", json={ @@ -393,16 +386,14 @@ def test_spendable_request_error(self): self.mock_wallet_service = MagicMock(WalletService) spendable_descriptor_mock = MagicMock(bdk.Descriptor) - spendable_descriptor_mock.as_string = MagicMock( - return_value="mock_descriptor") + spendable_descriptor_mock.as_string = MagicMock(return_value="mock_descriptor") self.mock_wallet_service.create_spendable_descriptor = MagicMock( return_value=spendable_descriptor_mock ) wallet_mock = MagicMock(bdk.Wallet) get_address_mock = MagicMock() address_mock = MagicMock(return_value=get_address_mock) - get_address_mock.address.as_string = MagicMock( - return_value="mock_address") + get_address_mock.address.as_string = MagicMock(return_value="mock_address") wallet_mock.get_address = address_mock self.mock_wallet_service.create_spendable_wallet = MagicMock( return_value=wallet_mock diff --git a/backend/src/tests/mocks.py b/backend/src/tests/mocks.py index ed1d5c14..e7d76aa2 100644 --- a/backend/src/tests/mocks.py +++ b/backend/src/tests/mocks.py @@ -5,6 +5,8 @@ import json import bdkpython as bdk +from src.my_types.transactions import LiveWalletOutput + tx_out_mock = bdk.TxOut(value=1000, script_pubkey="mock_script_pubkey") outpoint_mock = bdk.OutPoint(txid="txid", vout=0) @@ -38,7 +40,14 @@ mock_electrum_get_transactions_response["result"], strict=True ) +tx_mock = Transaction.parse( + mock_electrum_get_transactions_response["result"], strict=True +) + +all_transactions_mock = [tx_mock] -all_transactions_mock = [ - Transaction.parse(mock_electrum_get_transactions_response["result"], strict=True) +all_outputs_mock = [ + LiveWalletOutput( + annominity_set=1, base_output=tx_mock.outputs[0], txid="mock_txid", labels=[] + ) ] diff --git a/backend/src/tests/service_tests/test_wallet_service.py b/backend/src/tests/service_tests/test_wallet_service.py index ecf4bc32..7ff2d705 100644 --- a/backend/src/tests/service_tests/test_wallet_service.py +++ b/backend/src/tests/service_tests/test_wallet_service.py @@ -9,6 +9,7 @@ GetTransactionsResponse, ) from src.models.wallet import Wallet +from src.my_types.transactions import LiveWalletOutput from src.services import WalletService from src.services.wallet.wallet import ( GetFeeEstimateForUtxoResponseType, @@ -979,37 +980,83 @@ def test_get_all_outputs(self): self.wallet_service.wallet = mock_wallet mock_wallet.is_mine = Mock() # mark first output as mine and the second as not + annominity_set_count_mock = 2 mock_wallet.is_mine.side_effect = [True, False] self.wallet_service.get_all_transactions = Mock( return_value=all_transactions_mock ) + mock_annominity_sets = { + all_transactions_mock[0].outputs[0].value: annominity_set_count_mock, + all_transactions_mock[0].outputs[1].value: annominity_set_count_mock, + } + self.wallet_service.calculate_output_annominity_sets = Mock( + return_value=mock_annominity_sets + ) + mock_db_output_1 = Mock() + mock_db_output_1.id = 1 + mock_db_output_1.txid = all_transactions_mock[0].txid + mock_db_output_1.vout = all_transactions_mock[0].outputs[0].output_n + mock_db_output_1.labels = [] + + # This db output is ignored because of the is_mine call therefore we don't + # need to bother mocking all the values + mock_db_output_2 = Mock() + + mock_db_output_synced = [mock_db_output_1, mock_db_output_2] + self.wallet_service.sync_local_db_with_incoming_output = Mock( + side_effect=mock_db_output_synced + ) # call the method we are testing get_all_outputs_response = self.wallet_service.get_all_outputs() self.wallet_service.get_all_transactions.assert_called() + self.wallet_service.calculate_output_annominity_sets.assert_called() + self.wallet_service.sync_local_db_with_incoming_output.assert_called() + + # the returned outputs should only be the first one since we mocked out that the second output would return False when checking if it is mine assert mock_wallet.is_mine.call_count == 2 + assert len(get_all_outputs_response) == 1 - # the returned outputs should only be the first one since we mocked out that the second output would return False when checking if it is mine - assert get_all_outputs_response == [all_transactions_mock[0].outputs[0]] + assert get_all_outputs_response[0].annominity_set == 2 + assert get_all_outputs_response[0].txid == all_transactions_mock[0].txid + assert get_all_outputs_response[0].labels == [] def test_get_all_outputs_if_none_are_mine(self): mock_wallet = Mock() self.wallet_service.wallet = mock_wallet - mock_wallet.is_mine = Mock() - # mark all as False - mock_wallet.is_mine.return_value = False + mock_wallet.is_mine = Mock(return_value=False) + # mark No outputs as mine + annominity_set_count_mock = 2 self.wallet_service.get_all_transactions = Mock( return_value=all_transactions_mock ) + mock_annominity_sets = { + all_transactions_mock[0].outputs[0].value: annominity_set_count_mock, + all_transactions_mock[0].outputs[1].value: annominity_set_count_mock, + } + self.wallet_service.calculate_output_annominity_sets = Mock( + return_value=mock_annominity_sets + ) + + # These db output is ignored because of the is_mine call therefore we don't + # need to bother mocking all the values + + mock_db_output_1 = Mock() + mock_db_output_2 = Mock() + + mock_db_output_synced = [mock_db_output_1, mock_db_output_2] + self.wallet_service.sync_local_db_with_incoming_output = Mock( + side_effect=mock_db_output_synced + ) # call the method we are testing get_all_outputs_response = self.wallet_service.get_all_outputs() self.wallet_service.get_all_transactions.assert_called() + self.wallet_service.calculate_output_annominity_sets.assert_called() + self.wallet_service.sync_local_db_with_incoming_output.assert_called() assert mock_wallet.is_mine.call_count == 2 - - # no outputs are mine so an empty list should be returned assert get_all_outputs_response == [] From cca492e9245ed0e95a4230859151d948fb020c21 Mon Sep 17 00:00:00 2001 From: Joseph Wyman Date: Mon, 28 Oct 2024 19:53:27 -0400 Subject: [PATCH 18/85] add test for calculate_output_annominity_sets --- .../service_tests/test_wallet_service.py | 56 +++++++++++++++---- 1 file changed, 44 insertions(+), 12 deletions(-) diff --git a/backend/src/tests/service_tests/test_wallet_service.py b/backend/src/tests/service_tests/test_wallet_service.py index 7ff2d705..6823f496 100644 --- a/backend/src/tests/service_tests/test_wallet_service.py +++ b/backend/src/tests/service_tests/test_wallet_service.py @@ -1,5 +1,6 @@ from unittest.case import TestCase import os +import copy from unittest.mock import MagicMock, call, patch, Mock import pytest from src.api.electrum import ( @@ -25,6 +26,7 @@ local_utxo_mock, transaction_details_mock, all_transactions_mock, + tx_mock, ) from typing import cast @@ -109,7 +111,8 @@ def test_connect_wallet(self): descriptor_patch.assert_has_calls(expected_calls) database_config_memory_patch.assert_called() - block_chain_config_electrum_mock.assert_called_with(electrum_config_mock) + block_chain_config_electrum_mock.assert_called_with( + electrum_config_mock) electrum_config_patch.assert_called_with( wallet_details_mock.electrum_url, None, 2, 30, 100, True ) @@ -180,7 +183,8 @@ def test_connect_wallet_with_wallet_without_change_descriptor(self): descriptor_patch.assert_has_calls(expected_calls) database_config_memory_patch.assert_called() - block_chain_config_electrum_mock.assert_called_with(electrum_config_mock) + block_chain_config_electrum_mock.assert_called_with( + electrum_config_mock) electrum_config_patch.assert_called_with( wallet_details_mock.electrum_url, None, 2, 30, 100, True ) @@ -284,7 +288,8 @@ def test_connect_wallet_with_existing_wallet_but_differing_wallet_id_in_db( descriptor_patch.assert_has_calls(expected_calls) database_config_memory_patch.assert_called() - block_chain_config_electrum_mock.assert_called_with(electrum_config_mock) + block_chain_config_electrum_mock.assert_called_with( + electrum_config_mock) electrum_config_patch.assert_called_with( wallet_details_mock.electrum_url, None, 2, 30, 100, True ) @@ -333,7 +338,8 @@ def test_create_wallet(self): ): mock_wallet = MagicMock() wallet_model_patch.return_value = mock_wallet - wallet_model_patch.get_current_wallet = MagicMock(return_value=None) + wallet_model_patch.get_current_wallet = MagicMock( + return_value=None) add_mock = MagicMock() commit_mock = MagicMock() @@ -373,7 +379,8 @@ def test_create_wallet_with_no_change_descriptor(self): ): mock_wallet = MagicMock() wallet_model_patch.return_value = mock_wallet - wallet_model_patch.get_current_wallet = MagicMock(return_value=None) + wallet_model_patch.get_current_wallet = MagicMock( + return_value=None) add_mock = MagicMock() commit_mock = MagicMock() @@ -413,7 +420,8 @@ def test_create_wallet_with_existing_current_wallet_in_db(self): ): mock_wallet = MagicMock() wallet_model_patch.return_value = mock_wallet - wallet_model_patch.get_current_wallet = MagicMock(return_value=MagicMock) + wallet_model_patch.get_current_wallet = MagicMock( + return_value=MagicMock) add_mock = MagicMock() commit_mock = MagicMock() @@ -581,7 +589,8 @@ def test_build_transaction(self): output_count, ) - amount_in_each_output = (local_utxo_mock.txout.value / 2) / output_count + amount_in_each_output = ( + local_utxo_mock.txout.value / 2) / output_count tx_builder_mock.add_recipient.assert_called_with( script_mock, amount_in_each_output ) @@ -689,8 +698,10 @@ def test_get_fee_estimate_for_utxos(self): assert fee_estimate_response.status == "success" fee: int = cast(int, transaction_details_mock.fee) - expected_fee_percent = (fee / (transaction_details_mock.sent + fee)) * 100 - assert fee_estimate_response.data == FeeDetails(expected_fee_percent, fee) + expected_fee_percent = ( + fee / (transaction_details_mock.sent + fee)) * 100 + assert fee_estimate_response.data == FeeDetails( + expected_fee_percent, fee) assert fee_estimate_response.psbt == "mock_psbt" def test_get_fee_estimate_for_utxo_with_build_tx_unspendable(self): @@ -710,7 +721,8 @@ def test_get_fee_estimate_for_utxo_with_build_tx_unspendable(self): assert get_fee_estimate_response.data == None def test_get_fee_estimate_for_utxo_with_build_tx_error(self): - build_transaction_error_response = BuildTransactionResponseType("error", None) + build_transaction_error_response = BuildTransactionResponseType( + "error", None) with patch.object( WalletService, "build_transaction", @@ -855,7 +867,8 @@ def test_create_spendable_wallet(self): ) database_config_memory_patch.assert_called() - block_chain_config_electrum_mock.assert_called_with(electrum_config_mock) + block_chain_config_electrum_mock.assert_called_with( + electrum_config_mock) electrum_config_patch.assert_called_with( electrum_url_mock, None, 2, 30, 100, True ) @@ -1040,7 +1053,7 @@ def test_get_all_outputs_if_none_are_mine(self): return_value=mock_annominity_sets ) - # These db output is ignored because of the is_mine call therefore we don't + # These db outputs are ignored because of the is_mine call therefore we don't # need to bother mocking all the values mock_db_output_1 = Mock() @@ -1060,3 +1073,22 @@ def test_get_all_outputs_if_none_are_mine(self): assert mock_wallet.is_mine.call_count == 2 assert get_all_outputs_response == [] + + def test_calculate_output_annominity_sets(self): + first_output_value = tx_mock.outputs[0].value + second_output_value = tx_mock.outputs[1].value + first_output_copy = copy.deepcopy(tx_mock.outputs[0]) + mock_transaction_outputs = [ + tx_mock.outputs[0], + first_output_copy, + tx_mock.outputs[1], + ] + response = self.wallet_service.calculate_output_annominity_sets( + mock_transaction_outputs + ) + + # since the first output and the copy have the same value + # the set should be 2. + assert response[first_output_value] == 2 + # since the second output is unique it should only have a count of 1 + assert response[second_output_value] == 1 From acd475669bbe9369f42a98eb8d75acf4ceedc5cf Mon Sep 17 00:00:00 2001 From: Joseph Wyman Date: Mon, 28 Oct 2024 20:06:28 -0400 Subject: [PATCH 19/85] add test for get output modtest_get_output_labels --- .../service_tests/test_wallet_service.py | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/backend/src/tests/service_tests/test_wallet_service.py b/backend/src/tests/service_tests/test_wallet_service.py index 6823f496..f7416d8e 100644 --- a/backend/src/tests/service_tests/test_wallet_service.py +++ b/backend/src/tests/service_tests/test_wallet_service.py @@ -10,6 +10,7 @@ GetTransactionsResponse, ) from src.models.wallet import Wallet +from src.my_types.controller_types.utxos_dtos import OutputLabelDto from src.my_types.transactions import LiveWalletOutput from src.services import WalletService from src.services.wallet.wallet import ( @@ -1092,3 +1093,29 @@ def test_calculate_output_annominity_sets(self): assert response[first_output_value] == 2 # since the second output is unique it should only have a count of 1 assert response[second_output_value] == 1 + + def test_get_output_labels(self): + with patch("src.services.wallet.wallet.Label") as label_model_patch: + mock_label_one = Mock() + mock_label_one.name = "label_one" + mock_label_one.display_name = "display_one" + mock_label_one.description = "description_one" + + mock_label_two = Mock() + mock_label_two.name = "label_two" + mock_label_two.display_name = "display_two" + mock_label_two.description = "description_two" + label_mocks = [mock_label_one, mock_label_two] + label_model_patch.query.all = Mock(return_value=label_mocks) + + response = self.wallet_service.get_output_labels() + assert isinstance(response[0], OutputLabelDto) + assert isinstance(response[1], OutputLabelDto) + + assert response[0].label == "label_one" + assert response[0].display_name == "display_one" + assert response[0].description == "description_one" + + assert response[1].label == "label_two" + assert response[1].display_name == "display_two" + assert response[1].description == "description_two" From 511e5555a9fc458886caaac42e203b45b5757a3b Mon Sep 17 00:00:00 2001 From: Joseph Wyman Date: Tue, 29 Oct 2024 20:08:58 -0400 Subject: [PATCH 20/85] add test for get output label unique --- .../service_tests/test_wallet_service.py | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/backend/src/tests/service_tests/test_wallet_service.py b/backend/src/tests/service_tests/test_wallet_service.py index f7416d8e..adcd7237 100644 --- a/backend/src/tests/service_tests/test_wallet_service.py +++ b/backend/src/tests/service_tests/test_wallet_service.py @@ -1119,3 +1119,43 @@ def test_get_output_labels(self): assert response[1].label == "label_two" assert response[1].display_name == "display_two" assert response[1].description == "description_two" + + def test_get_output_labels_unique(self): + with patch("src.services.wallet.wallet.DB.session") as mock_db_session: + mock_label_one = Mock() + mock_label_one.name = "label_one" + mock_label_one.display_name = "display_one" + mock_label_one.description = "description_one" + + mock_label_two = Mock() + mock_label_two.name = "label_two" + mock_label_two.display_name = "display_two" + mock_label_two.description = "description_two" + mock_outputlabel = [ + Mock(txid="txid_one", vout=0, labels=[ + mock_label_one, mock_label_two]), + Mock(txid="txid_two", vout=1, labels=[mock_label_one]), + ] + + mock_query = mock_db_session.query.return_value + mock_query.join.return_value = mock_query + mock_query.group_by.return_value = mock_query + mock_query.having.return_value = mock_query + mock_query.all.return_value = mock_outputlabel + + response = self.wallet_service.get_output_labels_unique() + print("what r", response) + assert response["txid_one-0"][0].label == mock_label_one.name + assert response["txid_one-0"][0].display_name == mock_label_one.display_name + assert response["txid_one-0"][0].description == mock_label_one.description + + assert response["txid_one-0"][1].label == mock_label_two.name + assert response["txid_one-0"][1].display_name == mock_label_two.display_name + assert response["txid_one-0"][1].description == mock_label_two.description + + assert response["txid_two-1"][0].label == mock_label_one.name + assert response["txid_two-1"][0].display_name == mock_label_one.display_name + assert response["txid_two-1"][0].description == mock_label_one.description + + assert isinstance(response["txid_one-0"][0], OutputLabelDto) + assert isinstance(response["txid_one-0"][1], OutputLabelDto) From e9b879ff3d168dc442d520e4ea8759e2ec8c5838 Mon Sep 17 00:00:00 2001 From: Joseph Wyman Date: Wed, 30 Oct 2024 10:28:19 -0400 Subject: [PATCH 21/85] add additional label related service unit tests --- backend/src/services/wallet/wallet.py | 37 +++--- .../service_tests/test_wallet_service.py | 118 +++++++++++++++++- 2 files changed, 136 insertions(+), 19 deletions(-) diff --git a/backend/src/services/wallet/wallet.py b/backend/src/services/wallet/wallet.py index 4c7a5426..090c0ae7 100644 --- a/backend/src/services/wallet/wallet.py +++ b/backend/src/services/wallet/wallet.py @@ -161,7 +161,8 @@ def connect_wallet( ) wallet_change_descriptor = ( - bdk.Descriptor(change_descriptor, bdk.Network._value2member_map_[network]) + bdk.Descriptor(change_descriptor, + bdk.Network._value2member_map_[network]) if change_descriptor else None ) @@ -183,7 +184,8 @@ def connect_wallet( database_config=db_config, ) - LOGGER.info(f"Connecting a new wallet to electrum server {wallet_details_id}") + LOGGER.info( + f"Connecting a new wallet to electrum server {wallet_details_id}") LOGGER.info(f"xpub {wallet_descriptor.as_string()}") wallet.sync(blockchain, None) @@ -232,7 +234,8 @@ def create_spendable_descriptor( twelve_word_secret = bdk.Mnemonic(bdk.WordCount.WORDS12) # xpriv - descriptor_secret_key = bdk.DescriptorSecretKey(network, twelve_word_secret, "") + descriptor_secret_key = bdk.DescriptorSecretKey( + network, twelve_word_secret, "") wallet_descriptor = None if script_type == ScriptType.P2PKH: @@ -339,7 +342,8 @@ def get_all_outputs(self) -> List[LiveWalletOutput]: all_transactions = self.get_all_transactions() all_outputs: List[LiveWalletOutput] = [] for transaction in all_transactions: - annominity_sets = self.calculate_output_annominity_sets(transaction.outputs) + annominity_sets = self.calculate_output_annominity_sets( + transaction.outputs) for output in transaction.outputs: db_output = self.sync_local_db_with_incoming_output( txid=transaction.txid, vout=output.output_n @@ -359,6 +363,7 @@ def get_all_outputs(self) -> List[LiveWalletOutput]: return all_outputs + # TODO add a better name since this is just adding the output to the db def sync_local_db_with_incoming_output( self, txid: str, @@ -399,7 +404,7 @@ def calculate_output_annominity_sets( return output_count def get_output_labels(self) -> List[OutputLabelDto]: - """Get all the labels for the outputs in the wallet.""" + """Get all the possible labels.""" labels = Label.query.all() return [ OutputLabelDto( @@ -413,7 +418,7 @@ def get_output_labels(self) -> List[OutputLabelDto]: # TODO name this better def get_output_labels_unique( self, - ) -> Dict[str, OutputLabelDto]: + ) -> Dict[str, List[OutputLabelDto]]: """Get all the labels for the outputs in the wallet and return them as a dictionary of the key-id mapped to an array of labels. @@ -457,17 +462,11 @@ def populate_outputs_and_labels( model_dump = populate_output_labels.model_dump() for unique_output_txid_vout in model_dump.keys(): txid, vout = unique_output_txid_vout.split("-") - # db_output = OutputModel.query.filter_by(txid=txid, vout=vout).first() - # if db_output is None: - # # add the output to the db - # db_output = OutputModel(txid=txid, vout=vout, labels=[]) - # DB.session.add(db_output) - # DB.session.flush() - self.sync_local_db_with_incoming_output(txid, vout) + self.sync_local_db_with_incoming_output(txid, int(vout)) output_labels = model_dump[unique_output_txid_vout] for label in output_labels: display_name = label["display_name"] - self.add_label_to_output(txid, vout, display_name) + self.add_label_to_output(txid, int(vout), display_name) except Exception as e: LOGGER.error("Error populating outputs and labels", error=e) DB.session.rollback() @@ -499,8 +498,11 @@ def remove_label_from_output( return db_output.labels def get_utxos_info(self, utxos_wanted: List[bdk.OutPoint]) -> List[bdk.LocalUtxo]: - """For a given set of txids and the vout pointing to a utxo, return the utxos""" - existing_utxos = cast(List[bdk.LocalUtxo], self.get_all_utxos()) + """Get only the specified utxos from the users entire set of utxos. + + The wanted utxos are specificed via a list of Outpoints which are just the txid and vout. + """ + existing_utxos = self.get_all_utxos() utxo_dict = { f"{utxo.outpoint.txid}_{utxo.outpoint.vout}": utxo for utxo in existing_utxos @@ -554,7 +556,8 @@ def build_transaction( script, amount_per_recipient_output ) - built_transaction: bdk.TxBuilderResult = tx_builder.finish(self.wallet) + built_transaction: bdk.TxBuilderResult = tx_builder.finish( + self.wallet) built_transaction.transaction_details.transaction return BuildTransactionResponseType( diff --git a/backend/src/tests/service_tests/test_wallet_service.py b/backend/src/tests/service_tests/test_wallet_service.py index adcd7237..28d632f8 100644 --- a/backend/src/tests/service_tests/test_wallet_service.py +++ b/backend/src/tests/service_tests/test_wallet_service.py @@ -2,6 +2,7 @@ import os import copy from unittest.mock import MagicMock, call, patch, Mock +from src.models.outputs import Output as OutputModel import pytest from src.api.electrum import ( ElectrumMethod, @@ -10,7 +11,10 @@ GetTransactionsResponse, ) from src.models.wallet import Wallet -from src.my_types.controller_types.utxos_dtos import OutputLabelDto +from src.my_types.controller_types.utxos_dtos import ( + OutputLabelDto, + PopulateOutputLabelsRequestDto, +) from src.my_types.transactions import LiveWalletOutput from src.services import WalletService from src.services.wallet.wallet import ( @@ -29,7 +33,7 @@ all_transactions_mock, tx_mock, ) -from typing import cast +from typing import List, cast wallet_descriptor = os.getenv("WALLET_DESCRIPTOR", "") @@ -1159,3 +1163,113 @@ def test_get_output_labels_unique(self): assert isinstance(response["txid_one-0"][0], OutputLabelDto) assert isinstance(response["txid_one-0"][1], OutputLabelDto) + + def test_populate_outputs_and_labels(self): + mock_label_one = dict( + label="label_one", display_name="display_one", description="description_one" + ) + + mock_label_two = dict( + label="label_two", display_name="display_two", description="description_two" + ) + + output_labels_in_populate_format = ( + PopulateOutputLabelsRequestDto.model_validate( + { + "txidone-0": [mock_label_one, mock_label_two], + "txidone-1": [mock_label_one, mock_label_two], + "txidtwo-0": [mock_label_one], + } + ) + ) + mock_add_label_to_output = Mock(return_value=None) + self.wallet_service.add_label_to_output = mock_add_label_to_output + mock_sync_local_db_with_incoming_output = Mock(return_value=None) + self.wallet_service.sync_local_db_with_incoming_output = ( + mock_sync_local_db_with_incoming_output + ) + + # call the method we are testing + populate_outputs_and_labels_response = ( + self.wallet_service.populate_outputs_and_labels( + output_labels_in_populate_format + ) + ) + + # called once for each output + assert mock_sync_local_db_with_incoming_output.call_count == 3 + mock_sync_local_db_with_incoming_output.assert_any_call("txidone", 0) + mock_sync_local_db_with_incoming_output.assert_any_call("txidone", 1) + + mock_sync_local_db_with_incoming_output.assert_any_call("txidtwo", 0) + + mock_add_label_to_output.assert_any_call( + "txidone", 0, mock_label_one["display_name"] + ) + + mock_add_label_to_output.assert_any_call( + "txidone", 1, mock_label_two["display_name"] + ) + mock_add_label_to_output.assert_any_call( + "txidtwo", 0, mock_label_one["display_name"] + ) + + assert populate_outputs_and_labels_response == None + + def test_get_utxos_info(self): + mock_outpoint_one = Mock() + mock_outpoint_one.txid = "mock_txid_one" + mock_outpoint_one.vout = 0 + + mock_outpoint_two = Mock() + mock_outpoint_two.txid = "mock_txid_two" + mock_outpoint_two.vout = 0 + + mock_all_utxos_one = Mock() + mock_all_utxos_one.outpoint = mock_outpoint_one + + mock_all_utxos_two = Mock() + mock_all_utxos_two.outpoint = mock_outpoint_two + + mock_all_utxos_three = Mock() + mock_all_utxos_three.outpoint = Mock() + + all_utxos_mock = [mock_all_utxos_one, + mock_all_utxos_two, mock_all_utxos_three] + mock_get_all_utxos = Mock(return_value=all_utxos_mock) + self.wallet_service.get_all_utxos = mock_get_all_utxos + + utxos_wanted_mock = [mock_outpoint_one] + response = self.wallet_service.get_utxos_info( + cast(List[bdk.OutPoint], utxos_wanted_mock) + ) + assert response[0] == mock_all_utxos_one + + def test_sync_local_db_with_incoming_output_for_new_output(self): + with patch("src.services.wallet.wallet.OutputModel") as mock_output_model: + mock_new_output_model = Mock() + # return None, as if we did not find this OutputModel in the db + mock_output_model.query.filter_by.return_value.first.return_value = None + mock_add_output_to_db = Mock(return_value=mock_new_output_model) + + self.wallet_service.add_output_to_db = mock_add_output_to_db + response = self.wallet_service.sync_local_db_with_incoming_output( + "txid", 0) + mock_add_output_to_db.assert_called_with(txid="txid", vout=0) + assert response == mock_new_output_model + + def test_sync_local_db_with_incoming_output_for_existing_output(self): + with patch("src.services.wallet.wallet.OutputModel") as mock_output_model: + mock_existing_output_model = Mock() + # return None, as if we did not find this OutputModel in the db + mock_output_model.query.filter_by.return_value.first.return_value = ( + mock_existing_output_model + ) + mock_add_output_to_db = Mock() + + self.wallet_service.add_output_to_db = mock_add_output_to_db + response = self.wallet_service.sync_local_db_with_incoming_output( + "txid", 0) + # since the output already exists in the db we should not call add_output_to_db + mock_add_output_to_db.assert_not_called() + assert response == mock_existing_output_model From ddb774a2087b0a55db78c05698e31f7139897c93 Mon Sep 17 00:00:00 2001 From: Joseph Wyman Date: Thu, 31 Oct 2024 20:23:00 -0400 Subject: [PATCH 22/85] add initial privacy view in table --- NOTES.md | 2 + TODO.md | 2 + src/app/api/types.ts | 4 +- src/app/components/privacy/privacySvg.tsx | 24 +++ .../components/privacy/transactionsTable.tsx | 204 ++++++++++++++++++ src/app/pages/Privacy.tsx | 10 +- 6 files changed, 242 insertions(+), 4 deletions(-) create mode 100644 src/app/components/privacy/privacySvg.tsx create mode 100644 src/app/components/privacy/transactionsTable.tsx diff --git a/NOTES.md b/NOTES.md index 33570c68..90163d4b 100644 --- a/NOTES.md +++ b/NOTES.md @@ -826,6 +826,8 @@ What would the ideal utxo structure look like for privacy? - no-kyced. - maybe custom label - do I need the dust attack label? +- coinjoin change + - bad bank or uncoinjoined amount in a wasabi mix. # Initial thoughts on how v1 UI will look diff --git a/TODO.md b/TODO.md index b559da59..275b6041 100644 --- a/TODO.md +++ b/TODO.md @@ -9,6 +9,8 @@ # TODOs backend - add mypy +- add testing system that tests the database, allowing for more integration level tests, especially in services, I shouldn't have to mock out db calls in the services. + - using sqllite should be easy to setup and teardown the db each run. - add a pyproject.toml to manage ruff line length stuff so that ruff formatting is inline with the lsp? - Also set the default custom fee rate to the current rate diff --git a/src/app/api/types.ts b/src/app/api/types.ts index d93cd586..bb6d5155 100644 --- a/src/app/api/types.ts +++ b/src/app/api/types.ts @@ -166,8 +166,7 @@ export type PopulateOutputLabelsBodyType = { export type PopulateOutputLabelsResponse = { success: boolean; }; - -export type GetTransactionsResponseType = { +export type Transaction = { txid: string; date?: string; network: string; // e.g., "bitcoin" @@ -192,6 +191,7 @@ export type GetTransactionsResponseType = { verified: boolean; status: string; // e.g., "new" }; +export type GetTransactionsResponseType = { transactions: [Transaction] }; export type GetOutputsResponseType = { outputs: TransactionOutputType[]; diff --git a/src/app/components/privacy/privacySvg.tsx b/src/app/components/privacy/privacySvg.tsx new file mode 100644 index 00000000..475e0296 --- /dev/null +++ b/src/app/components/privacy/privacySvg.tsx @@ -0,0 +1,24 @@ +import React from 'react'; + +const PrivacyIcon = () => ( + + + + + + +); + +export default PrivacyIcon; diff --git a/src/app/components/privacy/transactionsTable.tsx b/src/app/components/privacy/transactionsTable.tsx new file mode 100644 index 00000000..91c6d896 --- /dev/null +++ b/src/app/components/privacy/transactionsTable.tsx @@ -0,0 +1,204 @@ +import React, { useMemo } from 'react'; + +import { createTheme, ThemeProvider } from '@mui/material'; +import { + MaterialReactTable, + useMaterialReactTable, +} from 'material-react-table'; + +import { CopyButton, rem, Tooltip, ActionIcon } from '@mantine/core'; + +import { + IconCheck, + IconCopy, +} from '@tabler/icons-react'; +import { BtcMetric, btcSatHandler } from '../../types/btcSatHandler'; +import { Transaction } from '../../api/types'; +import PrivacyIcon from './privacySvg'; + +const sectionColor = 'rgb(1, 67, 97)'; + +type TransactionsTableProps = { + transactions: [Transaction]; + btcMetric: BtcMetric; +}; + +export const TransactionsTable = ({ + transactions, + btcMetric, +}: TransactionsTableProps) => { + const columns = useMemo(() => { + const defaultColumns = [ + { + header: 'Analyze Privacy', + size: 50, + accessorKey: 'date', + Cell: ({ row }: { row: any }) => { + return ( +
+ {}} + > + + +
+ ); + }, + }, + { + header: 'Date', + size: 100, + accessorKey: 'date', + Cell: ({ row }: { row: any }) => { + return
{row.original.date || 'unknown'}
; + }, + }, + { + header: 'Txid', + accessorKey: 'txid', + size: 100, + Cell: ({ row }) => { + const prefix = row.original.txid.substring(0, 4); + const suffix = row.original.txid.substring( + row.original?.txid?.length - 4, + ); + const abrv = `${prefix}....${suffix}`; + return ( +
+ +

{abrv}

+ + + {({ copied, copy }) => ( + + + {copied ? ( + + ) : ( + + )} + + + )} + +
+ ); + }, + }, + // TODO add oclumn for my amount? + // add column for opening + // add additional values to transaction api response like + // how much I sent, how much I received, if a send, recieve or both. + { + header: 'Total Amount', + accessorKey: 'amount', + size: 100, + Cell: ({ row }: { row: any }) => { + const amount = btcSatHandler( + Number(row.original.output_total).toFixed(2).toLocaleString(), + btcMetric, + ); + + // TODO make red if sending transaction, green if receiving + return ( +
+

+ {btcMetric === BtcMetric.BTC + ? amount + : Number(amount).toLocaleString()} +

+
+ ); + }, + }, + ]; + + return defaultColumns; + }, [transactions, btcMetric]); + + const table = useMaterialReactTable({ + columns, + data: transactions, + enableRowSelection: false, + enableDensityToggle: false, + enableFullScreenToggle: false, + enableFilters: false, + enableColumnFilters: false, + enableColumnActions: false, + enableHiding: false, + enablePagination: false, + enableTableFooter: false, + enableBottomToolbar: false, + muiTableContainerProps: { + className: 'overflow-auto transition-all duration-300 ease-in-out', + style: { maxHeight: false ? '24rem' : '30rem' }, + }, + enableStickyHeader: true, + enableTopToolbar: true, + positionToolbarAlertBanner: 'none', + positionToolbarDropZone: 'top', + enableEditing: false, + renderTopToolbarCustomActions: ({ table }) => { + return ( +
+

+ Transactions +

+
+ ); + }, + muiSelectCheckboxProps: { + color: 'primary', + }, + + // @ts-ignore + muiTableBodyRowProps: { classes: { root: { after: 'bg-green-100' } } }, + }); + + return ( +
+ + + +
+ ); +}; diff --git a/src/app/pages/Privacy.tsx b/src/app/pages/Privacy.tsx index ceff7545..c41f631d 100644 --- a/src/app/pages/Privacy.tsx +++ b/src/app/pages/Privacy.tsx @@ -3,6 +3,7 @@ import { IconCoins, IconArrowsDownUp, IconEye } from '@tabler/icons-react'; import { useGetTransactions, useGetOutputs } from '../hooks/transactions'; import { TxosTable } from '../components/privacy/txosTable'; import { BtcMetric } from '../types/btcSatHandler'; +import { TransactionsTable } from '../components/privacy/transactionsTable'; type PrivacyProps = { btcMetric: BtcMetric; @@ -16,7 +17,7 @@ export const Privacy = ({ btcMetric }: PrivacyProps) => { } // TODO add back in and use when using the transaction tab - // const transactionsResponse = useGetTransactions(); + const transactionsResponse = useGetTransactions(); const outputs = useGetOutputs(); return (
@@ -59,7 +60,12 @@ export const Privacy = ({ btcMetric }: PrivacyProps) => { - My transactions +
+ +
My preview area From 0be766c934a81c34f1da7935131cafac621869e6 Mon Sep 17 00:00:00 2001 From: Joseph Wyman Date: Sat, 2 Nov 2024 10:18:29 -0400 Subject: [PATCH 23/85] add initial privacy and tx details modals from transction table --- .../components/TransactionDetailsModal.tsx | 30 ++++++++ .../components/TransactionPrivacyModal.tsx | 29 ++++++++ .../components/privacy/transactionsTable.tsx | 73 ++++++++++++++++--- 3 files changed, 121 insertions(+), 11 deletions(-) create mode 100644 src/app/components/TransactionDetailsModal.tsx create mode 100644 src/app/components/TransactionPrivacyModal.tsx diff --git a/src/app/components/TransactionDetailsModal.tsx b/src/app/components/TransactionDetailsModal.tsx new file mode 100644 index 00000000..cdebdfa8 --- /dev/null +++ b/src/app/components/TransactionDetailsModal.tsx @@ -0,0 +1,30 @@ +import React from 'react'; +import { Modal } from '@mantine/core'; +import { Transaction } from '../api/types'; +import { BtcMetric } from '../types/btcSatHandler'; +type TransactionDetailsModalProps = { + opened: boolean; + onClose: () => void; + transactionDetails: Transaction; + btcMetric: BtcMetric; +}; +export const TransactionDetailsModal = ({ + opened, + onClose, + transactionDetails, + btcMetric, +}: TransactionDetailsModalProps) => { + return ( + +
+

Transaction ID: {transactionDetails.txid}

+
+ + ); +}; diff --git a/src/app/components/TransactionPrivacyModal.tsx b/src/app/components/TransactionPrivacyModal.tsx new file mode 100644 index 00000000..e1457d27 --- /dev/null +++ b/src/app/components/TransactionPrivacyModal.tsx @@ -0,0 +1,29 @@ +import { Modal } from '@mantine/core'; +import { Transaction } from '../api/types'; +import { BtcMetric } from '../types/btcSatHandler'; +type TransactionDetailsModalProps = { + opened: boolean; + onClose: () => void; + transactionDetails: Transaction; + btcMetric: BtcMetric; +}; +export const TransactionPrivacyModal = ({ + opened, + onClose, + transactionDetails, + btcMetric, +}: TransactionDetailsModalProps) => { + return ( + +
+

Transaction ID: {transactionDetails.txid}

+
+ + ); +}; diff --git a/src/app/components/privacy/transactionsTable.tsx b/src/app/components/privacy/transactionsTable.tsx index 91c6d896..1050f434 100644 --- a/src/app/components/privacy/transactionsTable.tsx +++ b/src/app/components/privacy/transactionsTable.tsx @@ -1,4 +1,4 @@ -import React, { useMemo } from 'react'; +import React, { useMemo, useState } from 'react'; import { createTheme, ThemeProvider } from '@mui/material'; import { @@ -7,14 +7,13 @@ import { } from 'material-react-table'; import { CopyButton, rem, Tooltip, ActionIcon } from '@mantine/core'; +import { TbListDetails } from 'react-icons/tb'; -import { - IconCheck, - IconCopy, -} from '@tabler/icons-react'; +import { IconCheck, IconCopy } from '@tabler/icons-react'; import { BtcMetric, btcSatHandler } from '../../types/btcSatHandler'; import { Transaction } from '../../api/types'; import PrivacyIcon from './privacySvg'; +import { TransactionDetailsModal } from '../TransactionDetailsModal'; const sectionColor = 'rgb(1, 67, 97)'; @@ -27,20 +26,31 @@ export const TransactionsTable = ({ transactions, btcMetric, }: TransactionsTableProps) => { + const [isTransactionModalShowing, setIsTransactionModalShowing] = + useState(false); + const [selectedTransaction, setSelectedTransaction] = + useState(null); + + const [isPrivacyModalShowing, setIsPrivacyModalShowing] = useState(false); + const columns = useMemo(() => { const defaultColumns = [ { header: 'Analyze Privacy', size: 50, - accessorKey: 'date', + accessorKey: 'privacy', Cell: ({ row }: { row: any }) => { + const transactionDetails = row.original as Transaction; return ( -
+
{}} + onClick={() => { + setSelectedTransaction(transactionDetails); + setIsPrivacyModalShowing(true); + }} > @@ -48,6 +58,30 @@ export const TransactionsTable = ({ ); }, }, + + { + header: 'View details', + size: 50, + accessorKey: 'details', + Cell: ({ row }: { row: any }) => { + const transactionDetails = row.original as Transaction; + return ( +
+ { + setSelectedTransaction(transactionDetails); + setIsTransactionModalShowing(true); + }} + > + + +
+ ); + }, + }, { header: 'Date', size: 100, @@ -68,10 +102,10 @@ export const TransactionsTable = ({ const abrv = `${prefix}....${suffix}`; return (
- +

{abrv}

 - + {({ copied, copy }) => ( + + {isTransactionModalShowing && ( + setIsTransactionModalShowing(false)} + /> + )} + + {isPrivacyModalShowing && ( + setIsPrivacyModalShowing(false)} + /> + )}
); }; From 67a1951e88dd018546f1d45cb81bbd07f4a87a97 Mon Sep 17 00:00:00 2001 From: Joseph Wyman Date: Sat, 2 Nov 2024 10:43:46 -0400 Subject: [PATCH 24/85] set up initial privacy metric selection ui --- .../components/TransactionPrivacyModal.tsx | 54 ++++++++++++++++++- .../components/privacy/transactionsTable.tsx | 3 +- 2 files changed, 55 insertions(+), 2 deletions(-) diff --git a/src/app/components/TransactionPrivacyModal.tsx b/src/app/components/TransactionPrivacyModal.tsx index e1457d27..9d404f9b 100644 --- a/src/app/components/TransactionPrivacyModal.tsx +++ b/src/app/components/TransactionPrivacyModal.tsx @@ -1,6 +1,7 @@ -import { Modal } from '@mantine/core'; +import { Accordion, Checkbox, Modal } from '@mantine/core'; import { Transaction } from '../api/types'; import { BtcMetric } from '../types/btcSatHandler'; +import { useState } from 'react'; type TransactionDetailsModalProps = { opened: boolean; onClose: () => void; @@ -13,6 +14,56 @@ export const TransactionPrivacyModal = ({ transactionDetails, btcMetric, }: TransactionDetailsModalProps) => { + // TODO get from backend + const privacyMetrics = [ + { + value: 'Apples', + description: + 'Crisp and refreshing fruit. Apples are known for their versatility and nutritional benefits. They come in a variety of flavors and are great for snacking, baking, or adding to salads.', + }, + { + value: 'Bananas', + description: + 'Naturally sweet and potassium-rich fruit. Bananas are a popular choice for their energy-boosting properties and can be enjoyed as a quick snack, added to smoothies, or used in baking.', + }, + { + value: 'Broccoli', + description: + 'Nutrient-packed green vegetable. Broccoli is packed with vitamins, minerals, and fiber. It has a distinct flavor and can be enjoyed steamed, roasted, or added to stir-fries.', + }, + ]; + const [selectedMetrics, setSelectedMetrics] = useState([]); + const items = privacyMetrics.map((item) => ( + +
+
+ { + const newMetrics = [...selectedMetrics]; + + if (event.currentTarget.checked) { + newMetrics.push(item.value); + + setSelectedMetrics(newMetrics); + } else { + // remove the item from the array + const index = newMetrics.indexOf(item.value); + if (index > -1) { + newMetrics.splice(index, 1); + setSelectedMetrics(newMetrics); + } + } + }} + /> +
+
+ {item.value} + {item.description} +
+
+ + )); return (

Transaction ID: {transactionDetails.txid}

+ {items}
); diff --git a/src/app/components/privacy/transactionsTable.tsx b/src/app/components/privacy/transactionsTable.tsx index 1050f434..306a7863 100644 --- a/src/app/components/privacy/transactionsTable.tsx +++ b/src/app/components/privacy/transactionsTable.tsx @@ -14,6 +14,7 @@ import { BtcMetric, btcSatHandler } from '../../types/btcSatHandler'; import { Transaction } from '../../api/types'; import PrivacyIcon from './privacySvg'; import { TransactionDetailsModal } from '../TransactionDetailsModal'; +import { TransactionPrivacyModal } from '../TransactionPrivacyModal'; const sectionColor = 'rgb(1, 67, 97)'; @@ -243,7 +244,7 @@ export const TransactionsTable = ({ )} {isPrivacyModalShowing && ( - Date: Sat, 2 Nov 2024 18:26:05 -0400 Subject: [PATCH 25/85] add initial fetching and displaying of privacy metrics --- backend/src/app.py | 5 +- backend/src/controllers/__init__.py | 2 + backend/src/controllers/privacy_metrics.py | 47 +++++++++++++ backend/src/database.py | 31 +++++++++ backend/src/models/privacy_metric.py | 66 +++++++++++++++++++ backend/src/my_types/__init__.py | 6 ++ .../controller_types/privacy_metrics_dtos.py | 15 +++++ .../privacy_metrics/privacy_metrics.py | 0 backend/src/services/wallet/wallet.py | 2 +- src/app/api/api.ts | 11 ++++ src/app/api/types.ts | 12 +++- .../components/TransactionPrivacyModal.tsx | 15 +++-- src/app/hooks/privacyMetrics.ts | 16 +++++ 13 files changed, 219 insertions(+), 9 deletions(-) create mode 100644 backend/src/controllers/privacy_metrics.py create mode 100644 backend/src/models/privacy_metric.py create mode 100644 backend/src/my_types/controller_types/privacy_metrics_dtos.py create mode 100644 backend/src/services/privacy_metrics/privacy_metrics.py create mode 100644 src/app/hooks/privacyMetrics.ts diff --git a/backend/src/app.py b/backend/src/app.py index 915a81e3..290952ca 100644 --- a/backend/src/app.py +++ b/backend/src/app.py @@ -1,6 +1,6 @@ from flask import Flask, request from flask_cors import CORS -from src.database import DB, populate_labels +from src.database import DB, populate_labels, populate_privacy_metrics # initialize structlog from src.utils import logging # noqa: F401, E261 @@ -24,6 +24,7 @@ def create_app(cls) -> Flask: wallet_api, health_check_api, hardware_wallet_api, + privacy_metrics_api, ) from src.containers.service_container import ServiceContainer @@ -51,6 +52,7 @@ def create_app(cls) -> Flask: cls.app.register_blueprint(wallet_api) cls.app.register_blueprint(health_check_api) cls.app.register_blueprint(hardware_wallet_api) + cls.app.register_blueprint(privacy_metrics_api) return cls.app @@ -98,6 +100,7 @@ def setup_database(app): DB.create_all() populate_labels() + populate_privacy_metrics() # for some reason the frontend doesn't run the executable with app.y being __main__ diff --git a/backend/src/controllers/__init__.py b/backend/src/controllers/__init__.py index 89737eba..9c088876 100644 --- a/backend/src/controllers/__init__.py +++ b/backend/src/controllers/__init__.py @@ -7,3 +7,5 @@ from .health_check import health_check_api from .hardware_wallets import hardware_wallet_api + +from .privacy_metrics import privacy_metrics_api diff --git a/backend/src/controllers/privacy_metrics.py b/backend/src/controllers/privacy_metrics.py new file mode 100644 index 00000000..84ffe66e --- /dev/null +++ b/backend/src/controllers/privacy_metrics.py @@ -0,0 +1,47 @@ +from flask import Blueprint + +from src.database import DB +from src.models.privacy_metric import PrivacyMetric +from src.services import WalletService +from dependency_injector.wiring import inject, Provide +from src.containers.service_container import ServiceContainer +import structlog + +from src.my_types import GetAllPrivacyMetricsResponseDto, PrivacyMetricDto +from src.my_types.controller_types.generic_response_types import SimpleErrorResponse + +privacy_metrics_api = Blueprint( + "privacy_metrics", __name__, url_prefix="/privacy-metrics" +) + +LOGGER = structlog.get_logger() + + +@privacy_metrics_api.route("/") +@inject +def get_privacy_metrics( + wallet_service: WalletService = Provide[ServiceContainer.wallet_service], +): + """ + TODO + """ + try: + # TODO use a service + all_metrics = DB.session.query(PrivacyMetric).all() + + return GetAllPrivacyMetricsResponseDto.model_validate( + dict( + metrics=[ + PrivacyMetricDto( + name=privacy_metric.name, + display_name=privacy_metric.display_name, + description=privacy_metric.description, + ) + for privacy_metric in all_metrics + ] + ) + ).model_dump() + + except Exception as e: + LOGGER.error("error getting privacy metrics", error=e) + return SimpleErrorResponse(message="error getting privacy metrics").model_dump() diff --git a/backend/src/database.py b/backend/src/database.py index bebdcdc3..51d95272 100644 --- a/backend/src/database.py +++ b/backend/src/database.py @@ -32,3 +32,34 @@ def populate_labels(): except Exception as e: DB.session.rollback() raise e + + +def populate_privacy_metrics(): + try: + from src.models.privacy_metric import ( + PrivacyMetric, + PrivacyMetricName, + privacy_metrics_descriptions, + ) + + for privacy_name in PrivacyMetricName: + # Check if the privacy metric already exists + existing_privacy_metric = ( + DB.session.query(PrivacyMetric).filter_by( + name=privacy_name).first() + ) + + # If it does not exist, create a new one + if not existing_privacy_metric: + privacy_metric = PrivacyMetric( + name=privacy_name, + display_name=privacy_name.value, + description=privacy_metrics_descriptions[privacy_name], + ) + DB.session.add(privacy_metric) + + # Commit the session only once after all additions + DB.session.commit() + except Exception as e: + DB.session.rollback() + raise e diff --git a/backend/src/models/privacy_metric.py b/backend/src/models/privacy_metric.py new file mode 100644 index 00000000..31452e29 --- /dev/null +++ b/backend/src/models/privacy_metric.py @@ -0,0 +1,66 @@ +from sqlalchemy import String, Enum, Integer +from enum import Enum as PyEnum + + +from src.database import DB + + +class PrivacyMetricName(PyEnum): + # Minize personal information # TODO find better catagory name + ANNOMINITY_SET = "annominity set" + NO_ADDRESS_REUSE = "no address reuse" + MINIMAL_WEALTH_REVEAL = "minimal wealth reveal" + MINIMAL_TX_HISTORY_REVEAL = "minimal transaction history reveal" + TIMING_ANALYSIS = "timing analysis" + + # Minimize linkability of inputs and outputs through change detection. + NO_CHANGE = "no change" + NO_SMALL_CHANGE = "no small change" + NO_ROUND_NUMBER_PAYMENTS = "rcd no round number payments" + SAME_SCRIPT_TYPES = "rcd same script types" + AVOID_OUTPUT_SIZE_DIFFERENCE = "rcd avoid output size difference" + NO_UNNECESSARY_INPUT = "rcd no unnecessary input heuristic" + USE_MULTI_CHANGE_OUTPUTS = "rcd multi change outputs" + AVOID_COMMON_CHANGE_POSITION = "avoid common change position" + + # avoid combinding or using utxos that you do not want to be combined + NO_DO_NOT_SPEND_UTXOS = "do not spend do not spends" + NO_KYCED_UTXOS = "do not spend KYCED" + NO_DUST_ATTACK_UTXOS = "do not use dust attacks" + NO_POST_MIX_CHANGE = "no post mix change" + SEGREGATE_POSTMIX_AND_NONMIX = "segregate postmix and nonmix" + + # Add more labels as needed + + +privacy_metrics_descriptions = { + PrivacyMetricName.ANNOMINITY_SET: "A high anonymity set improves Bitcoin privacy by obscuring transaction origins and preventing address linking, making it harder for blockchain analysis to identify specific users and enhancing overall confidentiality.", + PrivacyMetricName.NO_ADDRESS_REUSE: "Not reusing addresses is good for privacy because it prevents transaction history from being easily linked to a single identity, making it more difficult for observers to trace user activity and associate multiple transactions with the same individual.", + PrivacyMetricName.MINIMAL_WEALTH_REVEAL: "Avoiding the use of UTXOs with large amounts is beneficial for privacy because it minimizes the risk of revealing your wealth and financial habits, making it harder for observers to assess your overall financial status or target you for theft or scams. This practice helps maintain a lower profile in transactions, reducing the likelihood of being linked to a specific identity or wealth level.", + PrivacyMetricName.MINIMAL_TX_HISTORY_REVEAL: "Avoiding the use of many UTXOs helps reduce the visibility of transaction history, as consolidating or minimizing UTXOs makes it harder for observers to track and link multiple transactions to a single identity. This practice enhances privacy by obscuring spending patterns and making it difficult to establish a comprehensive financial profile based on transaction behavior.", + PrivacyMetricName.TIMING_ANALYSIS: "Avoiding timing analysis in a Bitcoin transaction enhances privacy by obscuring the relationship between senders and receivers, making it harder for observers to link transactions to specific individuals. By randomizing transaction timing or using methods like coin mixing, users can protect their financial patterns and enhance overall anonymity within the network.", + PrivacyMetricName.NO_CHANGE: "Having no change in a Bitcoin transaction improves privacy by simplifying the transaction structure, making it harder for external observers to trace the flow of funds and link addresses back to their owners. This reduces the potential for address clustering, where multiple addresses are associated with the same user, thereby enhancing anonymity in the transaction history.", + PrivacyMetricName.NO_SMALL_CHANGE: "Avoiding small change in Bitcoin transactions enhances privacy by minimizing the number of unspent transaction outputs (UTXOs) associated with a user's wallet, which can be traced back to them. Small UTXOs often create identifiable patterns, revealing links between inputs and outputs, and making it easier for observers to track transaction histories and associate multiple addresses with a single user, thereby compromising anonymity.", + PrivacyMetricName.NO_ROUND_NUMBER_PAYMENTS: "Using non-round number payments in Bitcoin transactions helps obscure the identities of both the sender and receiver by reducing the likelihood of identifying who is involved in a transaction. Round number payments can create clear patterns, making it easier for observers to pinpoint the receiver based on the expected amounts, while irregular payment amounts complicate this analysis, making it difficult to discern the relationship between the parties and enhancing privacy for both.", + PrivacyMetricName.SAME_SCRIPT_TYPES: "Maintaining the same script type between inputs and outputs in a Bitcoin transaction enhances privacy by ensuring that all elements of the transaction appear uniform and consistent, making it more challenging for observers to draw conclusions about the transaction's structure. When inputs and outputs share the same script type, it becomes harder to differentiate between them, thereby obscuring the flow of funds and reducing the potential for address clustering, which can lead to a clearer association between addresses and their owners.", + PrivacyMetricName.AVOID_OUTPUT_SIZE_DIFFERENCE: "Having similar-sized outputs in Bitcoin transactions enhances privacy by making it more difficult for observers to link specific outputs to particular inputs, as uniform output sizes create ambiguity in tracing the flow of funds. This uniformity helps obscure the relationships between addresses, reducing the likelihood of clustering and making it harder for analysts to identify patterns or associate transactions with individual users, thereby bolstering overall anonymity.", + PrivacyMetricName.NO_UNNECESSARY_INPUT: "Reducing unnecessary inputs in a Bitcoin transaction is beneficial because it prevents the association of specific inputs with outputs, making it harder to trace the flow of funds. ", + PrivacyMetricName.USE_MULTI_CHANGE_OUTPUTS: "Having multiple outputs in a Bitcoin transaction enhances privacy by creating a more complex transaction structure, which obscures the relationship between inputs and outputs. This added complexity makes it difficult for observers to trace the flow of funds, thereby reducing the risk of linking specific inputs to identifiable outputs.", + PrivacyMetricName.AVOID_COMMON_CHANGE_POSITION: "Reusing the same change output (vout) position in Bitcoin transactions can lead to wallet clustering, where observers can group multiple addresses controlled by the same user based on transaction patterns. This compromises your privacy by making it easier to trace your spending behavior and link different transactions.", + PrivacyMetricName.NO_DO_NOT_SPEND_UTXOS: "Avoiding the use of 'do not spend' UTXOs in Bitcoin transactions is essential for maintaining privacy and security, as these UTXOs are typically flagged for specific purposes and should not be combined with other funds. By avoiding these UTXOs, you can prevent unintended disclosures of sensitive information and protect your financial privacy.", + PrivacyMetricName.NO_KYCED_UTXOS: "Avoiding the use of KYCed UTXOs in Bitcoin transactions is crucial for preserving privacy and anonymity, as these UTXOs are linked to your identity through Know Your Customer (KYC) processes. By excluding these UTXOs from your transactions, you can prevent the exposure of personal information and maintain a higher level of privacy and confidentiality.", + PrivacyMetricName.NO_DUST_ATTACK_UTXOS: "Avoiding the use of dust attack UTXOs in Bitcoin transactions is essential for protecting your privacy and security, as these UTXOs are often used to track and de-anonymize users through small, non-standard transactions. By excluding these UTXOs from your transactions, you can reduce the risk of being targeted by malicious actors and maintain a higher level of privacy and anonymity within the network.", + PrivacyMetricName.NO_POST_MIX_CHANGE: "Not using post mix change ensures that funds remain anonymized after mixing, preventing the re-identification of mixed funds and maintaining the privacy and anonymity provided by the mixing process. ", + PrivacyMetricName.SEGREGATE_POSTMIX_AND_NONMIX: "Segregating post-mix and non-mix funds in Bitcoin transactions is essential for maintaining privacy and anonymity, as combining these funds can undo the anonymization that the mixing provided.", +} + + +# +# +class PrivacyMetric(DB.Model): + __tablename__ = "privacy_metrics" # Specify the table name + + id = DB.Column(Integer, primary_key=True, autoincrement=True) + name = DB.Column(Enum(PrivacyMetricName), unique=True, nullable=False) + display_name = DB.Column(String, unique=True, nullable=False) + description = DB.Column(String, unique=True, nullable=False) diff --git a/backend/src/my_types/__init__.py b/backend/src/my_types/__init__.py index 0b18b1cf..c11b1a09 100644 --- a/backend/src/my_types/__init__.py +++ b/backend/src/my_types/__init__.py @@ -25,3 +25,9 @@ SimpleErrorResponse, ValidationErrorResponse, ) + + +from src.my_types.controller_types.privacy_metrics_dtos import ( + GetAllPrivacyMetricsResponseDto, + PrivacyMetricDto, +) diff --git a/backend/src/my_types/controller_types/privacy_metrics_dtos.py b/backend/src/my_types/controller_types/privacy_metrics_dtos.py new file mode 100644 index 00000000..134d2df9 --- /dev/null +++ b/backend/src/my_types/controller_types/privacy_metrics_dtos.py @@ -0,0 +1,15 @@ +from typing import Optional, List, Dict +from pydantic import BaseModel, RootModel, field_validator, Field +import structlog + +LOGGER = structlog.get_logger() + + +class PrivacyMetricDto(BaseModel): + name: str + display_name: str + description: str + + +class GetAllPrivacyMetricsResponseDto(BaseModel): + metrics: List[PrivacyMetricDto] diff --git a/backend/src/services/privacy_metrics/privacy_metrics.py b/backend/src/services/privacy_metrics/privacy_metrics.py new file mode 100644 index 00000000..e69de29b diff --git a/backend/src/services/wallet/wallet.py b/backend/src/services/wallet/wallet.py index 090c0ae7..69b2b07e 100644 --- a/backend/src/services/wallet/wallet.py +++ b/backend/src/services/wallet/wallet.py @@ -4,7 +4,7 @@ from sqlalchemy import func from src.models.label import Label from src.models.outputs import Output as OutputModel -from typing import Literal, Optional, cast, List, Dict +from typing import Literal, Optional, List, Dict from src.api import electrum_request, parse_electrum_url from src.api.electrum import ( diff --git a/src/app/api/api.ts b/src/app/api/api.ts index 61492431..241b10e9 100644 --- a/src/app/api/api.ts +++ b/src/app/api/api.ts @@ -25,6 +25,7 @@ import { GetOutputLabelsPopulateResponseType, PopulateOutputLabelsBodyType, PopulateOutputLabelsResponse, + GetPrivacyMetricsResponseType, } from './types'; import { Network } from '../types/network'; @@ -322,4 +323,14 @@ export class ApiClient { const data = (await response.json()) as GetBTCPriceResponseType; return data; } + + static async getAllPrivacyMetrics() { + const response = await fetchHandler( + `${configs.backendServerBaseUrl}/privacy-metrics`, + 'GET', + ); + + const data = (await response.json()) as GetPrivacyMetricsResponseType; + return data; + } } diff --git a/src/app/api/types.ts b/src/app/api/types.ts index bb6d5155..3d415131 100644 --- a/src/app/api/types.ts +++ b/src/app/api/types.ts @@ -214,5 +214,15 @@ export type RemoveLabelRequestParams = { }; export type RemoveLabelResponseType = { - labels: [OutputLabelType]; + labels: OutputLabelType[]; +}; + +export type PrivacyMetric = { + name: string; + display_name: string; + description: string; +}; + +export type GetPrivacyMetricsResponseType = { + metrics: PrivacyMetric[]; }; diff --git a/src/app/components/TransactionPrivacyModal.tsx b/src/app/components/TransactionPrivacyModal.tsx index 9d404f9b..95961e69 100644 --- a/src/app/components/TransactionPrivacyModal.tsx +++ b/src/app/components/TransactionPrivacyModal.tsx @@ -2,6 +2,7 @@ import { Accordion, Checkbox, Modal } from '@mantine/core'; import { Transaction } from '../api/types'; import { BtcMetric } from '../types/btcSatHandler'; import { useState } from 'react'; +import { useGetPrivacyMetrics } from '../hooks/privacyMetrics'; type TransactionDetailsModalProps = { opened: boolean; onClose: () => void; @@ -14,8 +15,9 @@ export const TransactionPrivacyModal = ({ transactionDetails, btcMetric, }: TransactionDetailsModalProps) => { + const getPRivacyMetricsResponse = useGetPrivacyMetrics(); // TODO get from backend - const privacyMetrics = [ + const privacyMetricsOne = [ { value: 'Apples', description: @@ -33,22 +35,23 @@ export const TransactionPrivacyModal = ({ }, ]; const [selectedMetrics, setSelectedMetrics] = useState([]); + const privacyMetrics = getPRivacyMetricsResponse?.data?.metrics || []; const items = privacyMetrics.map((item) => ( - +
{ const newMetrics = [...selectedMetrics]; if (event.currentTarget.checked) { - newMetrics.push(item.value); + newMetrics.push(item.name); setSelectedMetrics(newMetrics); } else { // remove the item from the array - const index = newMetrics.indexOf(item.value); + const index = newMetrics.indexOf(item.name); if (index > -1) { newMetrics.splice(index, 1); setSelectedMetrics(newMetrics); @@ -58,7 +61,7 @@ export const TransactionPrivacyModal = ({ />
- {item.value} + {item.display_name} {item.description}
diff --git a/src/app/hooks/privacyMetrics.ts b/src/app/hooks/privacyMetrics.ts new file mode 100644 index 00000000..3499cf96 --- /dev/null +++ b/src/app/hooks/privacyMetrics.ts @@ -0,0 +1,16 @@ +import { ApiClient } from '../api/api'; +import { useQuery } from 'react-query'; + +export const uxtoQueryKeys = { + getPrivacyMetrics: ['getPrivacyMetrics'], +}; + +export function useGetPrivacyMetrics() { + return useQuery( + uxtoQueryKeys.getPrivacyMetrics, + () => ApiClient.getAllPrivacyMetrics(), + { + refetchOnWindowFocus: true, + }, + ); +} From 6a5c73a3416395415d77f5e59013e62d6d76966e Mon Sep 17 00:00:00 2001 From: Joseph Wyman Date: Sun, 3 Nov 2024 09:55:03 -0500 Subject: [PATCH 26/85] implement initial tx privacy metric selection --- backend/src/controllers/privacy_metrics.py | 35 ++++- backend/src/my_types/__init__.py | 1 + .../controller_types/privacy_metrics_dtos.py | 13 +- src/app/api/api.ts | 15 ++ src/app/api/types.ts | 9 ++ .../components/TransactionPrivacyModal.tsx | 145 ++++++++++++------ src/app/hooks/privacyMetrics.ts | 22 ++- 7 files changed, 188 insertions(+), 52 deletions(-) diff --git a/backend/src/controllers/privacy_metrics.py b/backend/src/controllers/privacy_metrics.py index 84ffe66e..778c1e24 100644 --- a/backend/src/controllers/privacy_metrics.py +++ b/backend/src/controllers/privacy_metrics.py @@ -1,13 +1,22 @@ from flask import Blueprint +import json from src.database import DB from src.models.privacy_metric import PrivacyMetric +from src.my_types.controller_types.privacy_metrics_dtos import ( + AnalyzeTxPrivacyRequestDto, +) from src.services import WalletService from dependency_injector.wiring import inject, Provide from src.containers.service_container import ServiceContainer +from flask import request import structlog -from src.my_types import GetAllPrivacyMetricsResponseDto, PrivacyMetricDto +from src.my_types import ( + GetAllPrivacyMetricsResponseDto, + PrivacyMetricDto, + AnalyzeTxPrivacyResponseDto, +) from src.my_types.controller_types.generic_response_types import SimpleErrorResponse privacy_metrics_api = Blueprint( @@ -45,3 +54,27 @@ def get_privacy_metrics( except Exception as e: LOGGER.error("error getting privacy metrics", error=e) return SimpleErrorResponse(message="error getting privacy metrics").model_dump() + + +@privacy_metrics_api.route("/", methods=["POST"]) +@inject +def anaylze_tx_privacy( + wallet_service: WalletService = Provide[ServiceContainer.wallet_service], +): + """ + TODO + """ + try: + # TODO use a service to actually analyze the privacy + data = AnalyzeTxPrivacyRequestDto.model_validate(json.loads(request.data)) + print("todo do something with ", data) + + return AnalyzeTxPrivacyResponseDto.model_validate( + dict(results="mock results") + ).model_dump() + + except Exception as e: + LOGGER.error("error analzying transaction privacy metrics", error=e) + return SimpleErrorResponse( + message="error analzying transaction privacy metrics" + ).model_dump() diff --git a/backend/src/my_types/__init__.py b/backend/src/my_types/__init__.py index c11b1a09..947aca0f 100644 --- a/backend/src/my_types/__init__.py +++ b/backend/src/my_types/__init__.py @@ -30,4 +30,5 @@ from src.my_types.controller_types.privacy_metrics_dtos import ( GetAllPrivacyMetricsResponseDto, PrivacyMetricDto, + AnalyzeTxPrivacyResponseDto, ) diff --git a/backend/src/my_types/controller_types/privacy_metrics_dtos.py b/backend/src/my_types/controller_types/privacy_metrics_dtos.py index 134d2df9..1756471a 100644 --- a/backend/src/my_types/controller_types/privacy_metrics_dtos.py +++ b/backend/src/my_types/controller_types/privacy_metrics_dtos.py @@ -1,5 +1,5 @@ -from typing import Optional, List, Dict -from pydantic import BaseModel, RootModel, field_validator, Field +from typing import List +from pydantic import BaseModel import structlog LOGGER = structlog.get_logger() @@ -13,3 +13,12 @@ class PrivacyMetricDto(BaseModel): class GetAllPrivacyMetricsResponseDto(BaseModel): metrics: List[PrivacyMetricDto] + + +class AnalyzeTxPrivacyResponseDto(BaseModel): + results: str # TODO actually implement this + + +class AnalyzeTxPrivacyRequestDto(BaseModel): + txid: str + privacy_metrics: List[str] diff --git a/src/app/api/api.ts b/src/app/api/api.ts index 241b10e9..018c4088 100644 --- a/src/app/api/api.ts +++ b/src/app/api/api.ts @@ -26,6 +26,8 @@ import { PopulateOutputLabelsBodyType, PopulateOutputLabelsResponse, GetPrivacyMetricsResponseType, + AnalyzeTxPrivacyResponseType, + AnalyzeTxPrivacyRequestBody, } from './types'; import { Network } from '../types/network'; @@ -333,4 +335,17 @@ export class ApiClient { const data = (await response.json()) as GetPrivacyMetricsResponseType; return data; } + + static async analyzeTxPrivacy( + analyzeTxPrivacyRequestBody: AnalyzeTxPrivacyRequestBody, + ) { + const response = await fetchHandler( + `${configs.backendServerBaseUrl}/privacy-metrics`, + 'POST', + analyzeTxPrivacyRequestBody, + ); + + const data = (await response.json()) as AnalyzeTxPrivacyResponseType; + return data; + } } diff --git a/src/app/api/types.ts b/src/app/api/types.ts index 3d415131..5128783f 100644 --- a/src/app/api/types.ts +++ b/src/app/api/types.ts @@ -226,3 +226,12 @@ export type PrivacyMetric = { export type GetPrivacyMetricsResponseType = { metrics: PrivacyMetric[]; }; + +export type AnalyzeTxPrivacyRequestBody = { + txid: string; + privacy_metrics: string[]; +}; + +export type AnalyzeTxPrivacyResponseType = { + results: string; +}; diff --git a/src/app/components/TransactionPrivacyModal.tsx b/src/app/components/TransactionPrivacyModal.tsx index 95961e69..1accd85e 100644 --- a/src/app/components/TransactionPrivacyModal.tsx +++ b/src/app/components/TransactionPrivacyModal.tsx @@ -1,8 +1,11 @@ -import { Accordion, Checkbox, Modal } from '@mantine/core'; -import { Transaction } from '../api/types'; +import { Accordion, Button, Checkbox, Modal } from '@mantine/core'; +import { PrivacyMetric, Transaction } from '../api/types'; import { BtcMetric } from '../types/btcSatHandler'; import { useState } from 'react'; -import { useGetPrivacyMetrics } from '../hooks/privacyMetrics'; +import { + useAnalyzeTxPrivacy, + useGetPrivacyMetrics, +} from '../hooks/privacyMetrics'; type TransactionDetailsModalProps = { opened: boolean; onClose: () => void; @@ -16,69 +19,115 @@ export const TransactionPrivacyModal = ({ btcMetric, }: TransactionDetailsModalProps) => { const getPRivacyMetricsResponse = useGetPrivacyMetrics(); - // TODO get from backend - const privacyMetricsOne = [ - { - value: 'Apples', - description: - 'Crisp and refreshing fruit. Apples are known for their versatility and nutritional benefits. They come in a variety of flavors and are great for snacking, baking, or adding to salads.', - }, - { - value: 'Bananas', - description: - 'Naturally sweet and potassium-rich fruit. Bananas are a popular choice for their energy-boosting properties and can be enjoyed as a quick snack, added to smoothies, or used in baking.', - }, - { - value: 'Broccoli', - description: - 'Nutrient-packed green vegetable. Broccoli is packed with vitamins, minerals, and fiber. It has a distinct flavor and can be enjoyed steamed, roasted, or added to stir-fries.', - }, - ]; const [selectedMetrics, setSelectedMetrics] = useState([]); + const privacyMetrics = getPRivacyMetricsResponse?.data?.metrics || []; - const items = privacyMetrics.map((item) => ( - -
-
- { - const newMetrics = [...selectedMetrics]; + const privacyMetricsNames = privacyMetrics.map((item) => item.name); + const halfOfMetricsCount = Math.ceil(privacyMetrics.length / 2); + const firstHalf = privacyMetrics.slice(0, halfOfMetricsCount); + const secondHalf = privacyMetrics.slice(halfOfMetricsCount); + const analyzeTxPrivacyMutation = useAnalyzeTxPrivacy(); - if (event.currentTarget.checked) { - newMetrics.push(item.name); + const PrivacyMetricComponent = ({ + privacyMetric, + }: { + privacyMetric: PrivacyMetric; + }) => { + return ( + +
+
+ { + const newMetrics = [...selectedMetrics]; + + if (event.currentTarget.checked) { + newMetrics.push(privacyMetric.name); - setSelectedMetrics(newMetrics); - } else { - // remove the item from the array - const index = newMetrics.indexOf(item.name); - if (index > -1) { - newMetrics.splice(index, 1); setSelectedMetrics(newMetrics); + } else { + // remove the item from the array + const index = newMetrics.indexOf(privacyMetric.name); + if (index > -1) { + newMetrics.splice(index, 1); + setSelectedMetrics(newMetrics); + } } - } - }} - /> -
-
- {item.display_name} - {item.description} + }} + /> +
+
+ {privacyMetric.display_name} + {privacyMetric.description} +
-
- + + ); + }; + + const firstHalfMetrics = firstHalf.map((item) => ( + + )); + + const secondHalfMetrics = secondHalf.map((item) => ( + )); + + const analyzePrivacy = () => { + console.log('analyze privacy'); + analyzeTxPrivacyMutation.mutate({ + txid: transactionDetails.txid, + privacy_metrics: selectedMetrics, + }); + }; + + const areAllSelected = + selectedMetrics.length === privacyMetricsNames.length && + selectedMetrics.every( + (value, index) => value === privacyMetricsNames[index], + ); + return (

Transaction ID: {transactionDetails.txid}

- {items} +
+ { + if (areAllSelected) { + setSelectedMetrics([]); + } else { + setSelectedMetrics(privacyMetricsNames); + } + }} + size="lg" + label={

All

} + /> +
+
+ {firstHalfMetrics} + {secondHalfMetrics} +
+ ); }; diff --git a/src/app/hooks/privacyMetrics.ts b/src/app/hooks/privacyMetrics.ts index 3499cf96..65c960dc 100644 --- a/src/app/hooks/privacyMetrics.ts +++ b/src/app/hooks/privacyMetrics.ts @@ -1,5 +1,6 @@ import { ApiClient } from '../api/api'; -import { useQuery } from 'react-query'; +import { useMutation, useQuery } from 'react-query'; +import { AnalyzeTxPrivacyRequestBody } from '../api/types'; export const uxtoQueryKeys = { getPrivacyMetrics: ['getPrivacyMetrics'], @@ -14,3 +15,22 @@ export function useGetPrivacyMetrics() { }, ); } + +export function useAnalyzeTxPrivacy( + onSuccess?: () => void, + onError?: () => void, +) { + return useMutation( + (body: AnalyzeTxPrivacyRequestBody) => ApiClient.analyzeTxPrivacy(body), + { + onSuccess: () => { + if (onSuccess) { + onSuccess(); + } + }, + onError: () => { + onError(); + }, + }, + ); +} From e8279dd48757e86570b76e9d6de9289bbe14b199 Mon Sep 17 00:00:00 2001 From: Joseph Wyman Date: Sun, 3 Nov 2024 10:00:36 -0500 Subject: [PATCH 27/85] add commit --- backend/src/controllers/privacy_metrics.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/controllers/privacy_metrics.py b/backend/src/controllers/privacy_metrics.py index 778c1e24..b8a635ac 100644 --- a/backend/src/controllers/privacy_metrics.py +++ b/backend/src/controllers/privacy_metrics.py @@ -62,7 +62,7 @@ def anaylze_tx_privacy( wallet_service: WalletService = Provide[ServiceContainer.wallet_service], ): """ - TODO + TODO change """ try: # TODO use a service to actually analyze the privacy From 0fce19e0c3784485320c1e9b9ff18f14cc2b45ed Mon Sep 17 00:00:00 2001 From: Joseph Wyman Date: Sun, 3 Nov 2024 10:40:21 -0500 Subject: [PATCH 28/85] add initial outline of privacy service --- backend/src/containers/service_container.py | 8 +++++- backend/src/controllers/privacy_metrics.py | 27 +++++++++++-------- backend/src/services/__init__.py | 2 ++ .../privacy_metrics/privacy_metrics.py | 13 +++++++++ 4 files changed, 38 insertions(+), 12 deletions(-) diff --git a/backend/src/containers/service_container.py b/backend/src/containers/service_container.py index 5ed4e7d4..ed360f3f 100644 --- a/backend/src/containers/service_container.py +++ b/backend/src/containers/service_container.py @@ -1,11 +1,17 @@ from dependency_injector import containers, providers +from src.services.privacy_metrics.privacy_metrics import PrivacyMetricsService + class ServiceContainer(containers.DeclarativeContainer): from src.services import WalletService, FeeService, HardwareWalletService - wiring_config = containers.WiringConfiguration(packages=["..controllers", "..services"]) + wiring_config = containers.WiringConfiguration( + packages=["..controllers", "..services"] + ) wallet_service = providers.Factory(WalletService) hardware_wallet_service = providers.Singleton(HardwareWalletService) fee_service = providers.Factory(FeeService) + + privacy_metrics_service = providers.Factory(PrivacyMetricsService) diff --git a/backend/src/controllers/privacy_metrics.py b/backend/src/controllers/privacy_metrics.py index b8a635ac..2e6e35e2 100644 --- a/backend/src/controllers/privacy_metrics.py +++ b/backend/src/controllers/privacy_metrics.py @@ -1,12 +1,10 @@ from flask import Blueprint import json -from src.database import DB -from src.models.privacy_metric import PrivacyMetric from src.my_types.controller_types.privacy_metrics_dtos import ( AnalyzeTxPrivacyRequestDto, ) -from src.services import WalletService +from src.services import PrivacyMetricsService from dependency_injector.wiring import inject, Provide from src.containers.service_container import ServiceContainer from flask import request @@ -29,14 +27,16 @@ @privacy_metrics_api.route("/") @inject def get_privacy_metrics( - wallet_service: WalletService = Provide[ServiceContainer.wallet_service], + privacy_service: PrivacyMetricsService = Provide[ + ServiceContainer.privacy_metrics_service + ], ): """ - TODO + Get all privacy metrics. """ try: # TODO use a service - all_metrics = DB.session.query(PrivacyMetric).all() + all_metrics = privacy_service.get_all_privacy_metrics() return GetAllPrivacyMetricsResponseDto.model_validate( dict( @@ -59,15 +59,20 @@ def get_privacy_metrics( @privacy_metrics_api.route("/", methods=["POST"]) @inject def anaylze_tx_privacy( - wallet_service: WalletService = Provide[ServiceContainer.wallet_service], + privacy_service: PrivacyMetricsService = Provide[ + ServiceContainer.privacy_metrics_service + ], ): """ - TODO change + Analyze a selected transaction based on an array of selected privacy metrics. """ try: - # TODO use a service to actually analyze the privacy - data = AnalyzeTxPrivacyRequestDto.model_validate(json.loads(request.data)) - print("todo do something with ", data) + request_data = AnalyzeTxPrivacyRequestDto.model_validate( + json.loads(request.data) + ) + privacy_service.analyze_tx_privacy( + request_data.txid, request_data.privacy_metrics + ) return AnalyzeTxPrivacyResponseDto.model_validate( dict(results="mock results") diff --git a/backend/src/services/__init__.py b/backend/src/services/__init__.py index ff6d0650..fdbedb7b 100644 --- a/backend/src/services/__init__.py +++ b/backend/src/services/__init__.py @@ -2,3 +2,5 @@ from src.services.fees.fees import FeeService from src.services.hardware_wallet.hardware_wallet import HardwareWalletService + +from src.services.privacy_metrics.privacy_metrics import PrivacyMetricsService diff --git a/backend/src/services/privacy_metrics/privacy_metrics.py b/backend/src/services/privacy_metrics/privacy_metrics.py index e69de29b..9133ce80 100644 --- a/backend/src/services/privacy_metrics/privacy_metrics.py +++ b/backend/src/services/privacy_metrics/privacy_metrics.py @@ -0,0 +1,13 @@ +from src.database import DB +from src.models.privacy_metric import PrivacyMetric + + +class PrivacyMetricsService: + @classmethod + def get_all_privacy_metrics(self) -> list[PrivacyMetric]: + all_metrics = DB.session.query(PrivacyMetric).all() + return all_metrics + + @classmethod + def analyze_tx_privacy(self, txid: str, privacy_metrics: list[str]) -> str: + return "TODO" From 4ec2da7208d9f20429cc40de8661e7168074cb0d Mon Sep 17 00:00:00 2001 From: Joseph Wyman Date: Sun, 3 Nov 2024 11:43:45 -0500 Subject: [PATCH 29/85] add initial outline for getting tx privacy results --- backend/src/containers/service_container.py | 1 - backend/src/controllers/privacy_metrics.py | 4 +- backend/src/models/privacy_metric.py | 2 +- backend/src/my_types/__init__.py | 1 + .../controller_types/privacy_metrics_dtos.py | 12 +- .../privacy_metrics/privacy_metrics.py | 156 +++++++++++++++++- src/app/api/types.ts | 6 +- 7 files changed, 168 insertions(+), 14 deletions(-) diff --git a/backend/src/containers/service_container.py b/backend/src/containers/service_container.py index ed360f3f..9e0e2b83 100644 --- a/backend/src/containers/service_container.py +++ b/backend/src/containers/service_container.py @@ -13,5 +13,4 @@ class ServiceContainer(containers.DeclarativeContainer): wallet_service = providers.Factory(WalletService) hardware_wallet_service = providers.Singleton(HardwareWalletService) fee_service = providers.Factory(FeeService) - privacy_metrics_service = providers.Factory(PrivacyMetricsService) diff --git a/backend/src/controllers/privacy_metrics.py b/backend/src/controllers/privacy_metrics.py index 2e6e35e2..11290dd7 100644 --- a/backend/src/controllers/privacy_metrics.py +++ b/backend/src/controllers/privacy_metrics.py @@ -70,12 +70,12 @@ def anaylze_tx_privacy( request_data = AnalyzeTxPrivacyRequestDto.model_validate( json.loads(request.data) ) - privacy_service.analyze_tx_privacy( + results = privacy_service.analyze_tx_privacy( request_data.txid, request_data.privacy_metrics ) return AnalyzeTxPrivacyResponseDto.model_validate( - dict(results="mock results") + dict(results=results) ).model_dump() except Exception as e: diff --git a/backend/src/models/privacy_metric.py b/backend/src/models/privacy_metric.py index 31452e29..a95df087 100644 --- a/backend/src/models/privacy_metric.py +++ b/backend/src/models/privacy_metric.py @@ -5,7 +5,7 @@ from src.database import DB -class PrivacyMetricName(PyEnum): +class PrivacyMetricName(str, PyEnum): # Minize personal information # TODO find better catagory name ANNOMINITY_SET = "annominity set" NO_ADDRESS_REUSE = "no address reuse" diff --git a/backend/src/my_types/__init__.py b/backend/src/my_types/__init__.py index 947aca0f..2824f8ed 100644 --- a/backend/src/my_types/__init__.py +++ b/backend/src/my_types/__init__.py @@ -31,4 +31,5 @@ GetAllPrivacyMetricsResponseDto, PrivacyMetricDto, AnalyzeTxPrivacyResponseDto, + AnalyzeTxPrivacyRequestDto ) diff --git a/backend/src/my_types/controller_types/privacy_metrics_dtos.py b/backend/src/my_types/controller_types/privacy_metrics_dtos.py index 1756471a..fdb7e9e2 100644 --- a/backend/src/my_types/controller_types/privacy_metrics_dtos.py +++ b/backend/src/my_types/controller_types/privacy_metrics_dtos.py @@ -2,6 +2,8 @@ from pydantic import BaseModel import structlog +from src.models.privacy_metric import PrivacyMetricName + LOGGER = structlog.get_logger() @@ -15,10 +17,10 @@ class GetAllPrivacyMetricsResponseDto(BaseModel): metrics: List[PrivacyMetricDto] -class AnalyzeTxPrivacyResponseDto(BaseModel): - results: str # TODO actually implement this - - class AnalyzeTxPrivacyRequestDto(BaseModel): txid: str - privacy_metrics: List[str] + privacy_metrics: List[PrivacyMetricName] + + +class AnalyzeTxPrivacyResponseDto(BaseModel): + results: dict[PrivacyMetricName, bool] # TODO actually implement this diff --git a/backend/src/services/privacy_metrics/privacy_metrics.py b/backend/src/services/privacy_metrics/privacy_metrics.py index 9133ce80..3605a93d 100644 --- a/backend/src/services/privacy_metrics/privacy_metrics.py +++ b/backend/src/services/privacy_metrics/privacy_metrics.py @@ -1,13 +1,161 @@ from src.database import DB -from src.models.privacy_metric import PrivacyMetric +from src.models.privacy_metric import PrivacyMetric, PrivacyMetricName class PrivacyMetricsService: @classmethod - def get_all_privacy_metrics(self) -> list[PrivacyMetric]: + def get_all_privacy_metrics(cls) -> list[PrivacyMetric]: all_metrics = DB.session.query(PrivacyMetric).all() return all_metrics @classmethod - def analyze_tx_privacy(self, txid: str, privacy_metrics: list[str]) -> str: - return "TODO" + def analyze_tx_privacy( + cls, txid: str, privacy_metrics: list[PrivacyMetricName] + ) -> dict[PrivacyMetricName, bool]: + results: dict[PrivacyMetricName, bool] = dict() + for privacy_metric in privacy_metrics: + if privacy_metric == PrivacyMetricName.ANNOMINITY_SET: + mock_set = 5 + result = cls.analyze_annominit_set(txid, mock_set) + results[privacy_metric] = result + + elif privacy_metric == PrivacyMetricName.NO_ADDRESS_REUSE: + result = cls.analyze_no_address_reuse(txid) + results[privacy_metric] = result + elif privacy_metric == PrivacyMetricName.MINIMAL_WEALTH_REVEAL: + result = cls.analyze_minimal_wealth_reveal(txid) + results[privacy_metric] = result + + elif privacy_metric == PrivacyMetricName.MINIMAL_TX_HISTORY_REVEAL: + result = cls.analyze_minimal_tx_history_reveal(txid) + results[privacy_metric] = result + + elif privacy_metric == PrivacyMetricName.TIMING_ANALYSIS: + result = cls.analyze_timing_analysis(txid) + results[privacy_metric] = result + + elif privacy_metric == PrivacyMetricName.NO_CHANGE: + result = cls.analyze_no_change(txid) + results[privacy_metric] = result + + elif privacy_metric == PrivacyMetricName.NO_SMALL_CHANGE: + result = cls.analyze_no_small_change(txid) + results[privacy_metric] = result + + elif privacy_metric == PrivacyMetricName.NO_ROUND_NUMBER_PAYMENTS: + result = cls.analyze_no_round_number_payments(txid) + results[privacy_metric] = result + + elif privacy_metric == PrivacyMetricName.SAME_SCRIPT_TYPES: + result = cls.analyze_same_script_types(txid) + results[privacy_metric] = result + + elif privacy_metric == PrivacyMetricName.AVOID_OUTPUT_SIZE_DIFFERENCE: + result = cls.analyze_avoid_output_size_difference(txid) + results[privacy_metric] = result + + elif privacy_metric == PrivacyMetricName.NO_UNNECESSARY_INPUT: + result = cls.analyze_no_unnecessary_input(txid) + results[privacy_metric] = result + + elif privacy_metric == PrivacyMetricName.USE_MULTI_CHANGE_OUTPUTS: + result = cls.analyze_use_multi_change_outputs(txid) + results[privacy_metric] = result + + elif privacy_metric == PrivacyMetricName.AVOID_COMMON_CHANGE_POSITION: + result = cls.analyze_avoid_common_change_position(txid) + results[privacy_metric] = result + + elif privacy_metric == PrivacyMetricName.NO_DO_NOT_SPEND_UTXOS: + result = cls.analyze_no_do_not_spend_utxos(txid) + results[privacy_metric] = result + + elif privacy_metric == PrivacyMetricName.NO_KYCED_UTXOS: + result = cls.analyze_no_kyced_utxos(txid) + results[privacy_metric] = result + + elif privacy_metric == PrivacyMetricName.NO_DUST_ATTACK_UTXOS: + result = cls.analyze_no_dust_attack_utxos(txid) + results[privacy_metric] = result + + elif privacy_metric == PrivacyMetricName.NO_POST_MIX_CHANGE: + result = cls.analyze_no_post_mix_change(txid) + results[privacy_metric] = result + + elif privacy_metric == PrivacyMetricName.SEGREGATE_POSTMIX_AND_NONMIX: + result = cls.analyze_segregate_postmix_and_nonmix(txid) + results[privacy_metric] = result + + return results + + @classmethod + def analyze_annominit_set(cls, txid: str, desired_annominity_set: int) -> bool: + return True + + @classmethod + def analyze_no_address_reuse(cls, txid: str) -> bool: + return True + + @classmethod + def analyze_minimal_wealth_reveal(cls, txid: str) -> bool: + return True + + @classmethod + def analyze_minimal_tx_history_reveal(cls, txid: str) -> bool: + return True + + @classmethod + def analyze_timing_analysis(cls, txid: str) -> bool: + return True + + @classmethod + def analyze_no_change(cls, txid: str) -> bool: + return True + + @classmethod + def analyze_no_small_change(cls, txid: str) -> bool: + return True + + @classmethod + def analyze_no_round_number_payments(cls, txid: str) -> bool: + return True + + @classmethod + def analyze_same_script_types(cls, txid: str) -> bool: + return True + + @classmethod + def analyze_avoid_output_size_difference(cls, txid: str) -> bool: + return True + + @classmethod + def analyze_no_unnecessary_input(cls, txid: str) -> bool: + return True + + @classmethod + def analyze_use_multi_change_outputs(cls, txid: str) -> bool: + return True + + @classmethod + def analyze_avoid_common_change_position(cls, txid: str) -> bool: + return True + + @classmethod + def analyze_no_do_not_spend_utxos(cls, txid: str) -> bool: + return True + + @classmethod + def analyze_no_kyced_utxos(cls, txid: str) -> bool: + return True + + @classmethod + def analyze_no_dust_attack_utxos(cls, txid: str) -> bool: + return True + + @classmethod + def analyze_no_post_mix_change(cls, txid: str) -> bool: + return True + + @classmethod + def analyze_segregate_postmix_and_nonmix(cls, txid: str) -> bool: + return True diff --git a/src/app/api/types.ts b/src/app/api/types.ts index 5128783f..7b2ecded 100644 --- a/src/app/api/types.ts +++ b/src/app/api/types.ts @@ -232,6 +232,10 @@ export type AnalyzeTxPrivacyRequestBody = { privacy_metrics: string[]; }; +export type PrivacyResults = { + [key: string]: boolean; +}; + export type AnalyzeTxPrivacyResponseType = { - results: string; + results: PrivacyResults; }; From 5bb23d782c477d579caecb9cb77346d3e0f10ab6 Mon Sep 17 00:00:00 2001 From: Joseph Wyman Date: Mon, 4 Nov 2024 21:16:26 -0500 Subject: [PATCH 30/85] implement address reuse check and related refactoring --- backend/src/models/last_fetched.py | 17 + backend/src/models/outputs.py | 8 +- backend/src/services/__init__.py | 2 + .../last_fetched/last_fetched_service.py | 40 ++ .../privacy_metrics/privacy_metrics.py | 40 +- backend/src/services/wallet/wallet.py | 80 ++-- .../service_tests/test_wallet_service.py | 375 ++++++++++-------- 7 files changed, 368 insertions(+), 194 deletions(-) create mode 100644 backend/src/models/last_fetched.py create mode 100644 backend/src/services/last_fetched/last_fetched_service.py diff --git a/backend/src/models/last_fetched.py b/backend/src/models/last_fetched.py new file mode 100644 index 00000000..2beec075 --- /dev/null +++ b/backend/src/models/last_fetched.py @@ -0,0 +1,17 @@ +from sqlalchemy import Enum, Integer +from enum import Enum as PyEnum + + +from src.database import DB + + +class LastFetchedType(PyEnum): + OUTPUTS = "outputs" + + +class LastFetched(DB.Model): + __tablename__ = "last_fetched" + + id = DB.Column(Integer, primary_key=True, autoincrement=True) + type = DB.Column(Enum(LastFetchedType), unique=True, nullable=False) + timestamp = DB.Column(DB.DateTime, nullable=False, default=None) diff --git a/backend/src/models/outputs.py b/backend/src/models/outputs.py index 498b4b50..d467b061 100644 --- a/backend/src/models/outputs.py +++ b/backend/src/models/outputs.py @@ -16,8 +16,12 @@ class Output(DB.Model): ) vout = DB.Column(DB.Integer, nullable=False, default=0) + address = DB.Column(DB.String(), nullable=False) + # Relationship to labels - labels = DB.relationship("Label", secondary=output_labels, back_populates="outputs") + labels = DB.relationship( + "Label", secondary=output_labels, back_populates="outputs") # Unique constraint on the combination of txid and vout - __table_args__ = (DB.UniqueConstraint("txid", "vout", name="uq_txid_vout"),) + __table_args__ = (DB.UniqueConstraint( + "txid", "vout", name="uq_txid_vout"),) diff --git a/backend/src/services/__init__.py b/backend/src/services/__init__.py index fdbedb7b..17e4537e 100644 --- a/backend/src/services/__init__.py +++ b/backend/src/services/__init__.py @@ -4,3 +4,5 @@ from src.services.hardware_wallet.hardware_wallet import HardwareWalletService from src.services.privacy_metrics.privacy_metrics import PrivacyMetricsService + +from src.services.last_fetched.last_fetched_service import LastFetchedService diff --git a/backend/src/services/last_fetched/last_fetched_service.py b/backend/src/services/last_fetched/last_fetched_service.py new file mode 100644 index 00000000..f80df17c --- /dev/null +++ b/backend/src/services/last_fetched/last_fetched_service.py @@ -0,0 +1,40 @@ +from src.models.last_fetched import LastFetched, LastFetchedType +from datetime import datetime +from src.database import DB +from typing import Optional + +import structlog + +LOGGER = structlog.get_logger() + + +class LastFetchedService: + @classmethod + def update_last_fetched_outputs_type( + self, + ) -> None: + """Update the last fetched time for the outputs.""" + timestamp = datetime.now() + current_last_fetched_output = LastFetched.query.filter_by( + type=LastFetchedType.OUTPUTS + ).first() + if current_last_fetched_output: + current_last_fetched_output.timestamp = timestamp + else: + last_fetched_output = LastFetched( + type=LastFetchedType.OUTPUTS, timestamp=timestamp + ) + DB.session.add(last_fetched_output) + DB.session.commit() + + @classmethod + def get_last_fetched_output_datetime( + self, + ) -> Optional[datetime]: + """Get the last fetched time for the outputs.""" + last_fetched_output = LastFetched.query.filter_by( + type=LastFetchedType.OUTPUTS + ).first() + if last_fetched_output: + return last_fetched_output.timestamp + return None diff --git a/backend/src/services/privacy_metrics/privacy_metrics.py b/backend/src/services/privacy_metrics/privacy_metrics.py index 3605a93d..77cf26ed 100644 --- a/backend/src/services/privacy_metrics/privacy_metrics.py +++ b/backend/src/services/privacy_metrics/privacy_metrics.py @@ -1,5 +1,12 @@ from src.database import DB from src.models.privacy_metric import PrivacyMetric, PrivacyMetricName +from src.models.outputs import Output as OutputModel +from datetime import datetime, timedelta + + +import structlog + +LOGGER = structlog.get_logger() class PrivacyMetricsService: @@ -93,11 +100,40 @@ def analyze_annominit_set(cls, txid: str, desired_annominity_set: int) -> bool: return True @classmethod - def analyze_no_address_reuse(cls, txid: str) -> bool: + def analyze_no_address_reuse( + cls, + txid: str, + ) -> bool: + # can I inject this in? + # circular imports are currently preventing it. + from src.services.wallet.wallet import WalletService + from src.services.last_fetched.last_fetched_service import LastFetchedService + + # Check that all outputs have already been fetched recently + last_fetched_output_datetime = ( + LastFetchedService.get_last_fetched_output_datetime() + ) + now = datetime.now() + refetch_interval = timedelta(minutes=5) + should_refetch_outputs = now - last_fetched_output_datetime > refetch_interval + + if last_fetched_output_datetime is None or should_refetch_outputs: + LOGGER.info("No last fetched output datetime found, fetching all outputs") + # this will get all the outputs and add them to the database, ensuring that they exist + WalletService.get_all_outputs() + outputs = OutputModel.query.filter_by(txid=txid).all() + for output in outputs: + if WalletService.is_address_reused(output.address): + LOGGER.info(f"Address {output.address} has been reused") + return False + return True @classmethod - def analyze_minimal_wealth_reveal(cls, txid: str) -> bool: + def analyze_minimal_wealth_reveal( + cls, + txid: str, + ) -> bool: return True @classmethod diff --git a/backend/src/services/wallet/wallet.py b/backend/src/services/wallet/wallet.py index 69b2b07e..5b7929ee 100644 --- a/backend/src/services/wallet/wallet.py +++ b/backend/src/services/wallet/wallet.py @@ -23,6 +23,7 @@ PopulateOutputLabelsRequestDto, ) from src.my_types.transactions import LiveWalletOutput +from src.services.last_fetched.last_fetched_service import LastFetchedService from src.services.wallet.raw_output_script_examples import ( p2pkh_raw_output_script, p2pk_raw_output_script, @@ -161,8 +162,7 @@ def connect_wallet( ) wallet_change_descriptor = ( - bdk.Descriptor(change_descriptor, - bdk.Network._value2member_map_[network]) + bdk.Descriptor(change_descriptor, bdk.Network._value2member_map_[network]) if change_descriptor else None ) @@ -184,8 +184,7 @@ def connect_wallet( database_config=db_config, ) - LOGGER.info( - f"Connecting a new wallet to electrum server {wallet_details_id}") + LOGGER.info(f"Connecting a new wallet to electrum server {wallet_details_id}") LOGGER.info(f"xpub {wallet_descriptor.as_string()}") wallet.sync(blockchain, None) @@ -234,8 +233,7 @@ def create_spendable_descriptor( twelve_word_secret = bdk.Mnemonic(bdk.WordCount.WORDS12) # xpriv - descriptor_secret_key = bdk.DescriptorSecretKey( - network, twelve_word_secret, "") + descriptor_secret_key = bdk.DescriptorSecretKey(network, twelve_word_secret, "") wallet_descriptor = None if script_type == ScriptType.P2PKH: @@ -291,12 +289,13 @@ def get_all_utxos(self) -> List[bdk.LocalUtxo]: utxos = self.wallet.list_unspent() return utxos + @classmethod def get_all_transactions( - self, + cls, ) -> List[Transaction]: """Get all transactions for the current wallet.""" wallet_details = Wallet.get_current_wallet() - if self.wallet is None or wallet_details is None: + if cls.wallet is None or wallet_details is None: LOGGER.error("No electrum wallet or wallet details found.") return [] @@ -311,7 +310,7 @@ def get_all_transactions( LOGGER.error("No electrum url or port found in the wallet details") return [] - transactions = self.wallet.list_transactions(False) + transactions = cls.wallet.list_transactions(False) all_tx_details: List[Transaction] = [] @@ -332,24 +331,27 @@ def get_all_transactions( all_tx_details.append(electrum_response.data) return all_tx_details - def get_all_outputs(self) -> List[LiveWalletOutput]: + @classmethod + def get_all_outputs(cls) -> List[LiveWalletOutput]: """Get all spent and unspent transaction outputs for the current wallet and mutate them as needed. Calculate the annominity set for each output. Attach the txid to each output. Attach all labels to each output. Sync the database with the incoming outputs. """ - all_transactions = self.get_all_transactions() + all_transactions = cls.get_all_transactions() all_outputs: List[LiveWalletOutput] = [] for transaction in all_transactions: - annominity_sets = self.calculate_output_annominity_sets( - transaction.outputs) + annominity_sets = cls.calculate_output_annominity_sets(transaction.outputs) for output in transaction.outputs: - db_output = self.sync_local_db_with_incoming_output( - txid=transaction.txid, vout=output.output_n - ) script = bdk.Script(output.script.raw) - if self.wallet and self.wallet.is_mine(script): + if cls.wallet and cls.wallet.is_mine(script): + db_output = cls.sync_local_db_with_incoming_output( + txid=transaction.txid, + vout=output.output_n, + address=output.address, + ) + LastFetchedService.update_last_fetched_outputs_type() annominity_set = annominity_sets.get(output.value, 1) extended_output = LiveWalletOutput( @@ -364,27 +366,33 @@ def get_all_outputs(self) -> List[LiveWalletOutput]: return all_outputs # TODO add a better name since this is just adding the output to the db + @classmethod def sync_local_db_with_incoming_output( - self, + cls, txid: str, vout: int, + address: str, ) -> OutputModel: """Sync the local database with the incoming output. If the output is not in the database, add it. """ - db_output = OutputModel.query.filter_by(txid=txid, vout=vout).first() + db_output = OutputModel.query.filter_by( + txid=txid, vout=vout, address=address + ).first() if not db_output: - db_output = self.add_output_to_db(txid=txid, vout=vout) + db_output = cls.add_output_to_db(txid=txid, vout=vout, address=address) return db_output - def add_output_to_db(self, vout: int, txid: str) -> OutputModel: - db_output = OutputModel(txid=txid, vout=vout, labels=[]) + @classmethod + def add_output_to_db(cls, vout: int, txid: str, address: str) -> OutputModel: + db_output = OutputModel(txid=txid, vout=vout, address=address, labels=[]) DB.session.add(db_output) DB.session.commit() return db_output + @classmethod def calculate_output_annominity_sets( self, transaction_outputs: List[Output] ) -> dict[str, int]: # -> {"value": count } @@ -435,7 +443,7 @@ def get_output_labels_unique( for output in outputs_with_label: for label in output.labels: - key = f"{output.txid}-{output.vout}" + key = f"{output.txid}-{output.vout}-{output.address}" if result.get(key, None) is None: result[key] = [ OutputLabelDto( @@ -455,25 +463,27 @@ def get_output_labels_unique( return result + @classmethod def populate_outputs_and_labels( - self, populate_output_labels: PopulateOutputLabelsRequestDto + cls, populate_output_labels: PopulateOutputLabelsRequestDto ) -> None: # TODO maybe a success of fail reutn type? try: model_dump = populate_output_labels.model_dump() for unique_output_txid_vout in model_dump.keys(): - txid, vout = unique_output_txid_vout.split("-") - self.sync_local_db_with_incoming_output(txid, int(vout)) + txid, vout, address = unique_output_txid_vout.split("-") + cls.sync_local_db_with_incoming_output(txid, int(vout), address) output_labels = model_dump[unique_output_txid_vout] for label in output_labels: display_name = label["display_name"] - self.add_label_to_output(txid, int(vout), display_name) + cls.add_label_to_output(txid, int(vout), display_name) except Exception as e: LOGGER.error("Error populating outputs and labels", error=e) DB.session.rollback() # TODO should this even go here or in its own service? + @classmethod def add_label_to_output( - self, txid: str, vout: int, label_display_name: str + cls, txid: str, vout: int, label_display_name: str ) -> list[Label]: """Add a label to an output in the db.""" db_output = OutputModel.query.filter_by(txid=txid, vout=vout).first() @@ -556,8 +566,7 @@ def build_transaction( script, amount_per_recipient_output ) - built_transaction: bdk.TxBuilderResult = tx_builder.finish( - self.wallet) + built_transaction: bdk.TxBuilderResult = tx_builder.finish(self.wallet) built_transaction.transaction_details.transaction return BuildTransactionResponseType( @@ -632,3 +641,14 @@ def get_fee_estimate_for_utxos_from_request( ) return fee_estimate_response + + @classmethod + def is_address_reused(self, address: str) -> bool: + """Check if the address has been used in the wallet more than once.""" + outputs_with_this_address = OutputModel.query.filter_by(address=address).all() + address_used_count = len(outputs_with_this_address) + + if address_used_count > 1: + return True + else: + return False diff --git a/backend/src/tests/service_tests/test_wallet_service.py b/backend/src/tests/service_tests/test_wallet_service.py index 28d632f8..b5ed75f9 100644 --- a/backend/src/tests/service_tests/test_wallet_service.py +++ b/backend/src/tests/service_tests/test_wallet_service.py @@ -17,6 +17,7 @@ ) from src.my_types.transactions import LiveWalletOutput from src.services import WalletService +from src.services.last_fetched.last_fetched_service import LastFetchedService from src.services.wallet.wallet import ( GetFeeEstimateForUtxoResponseType, BuildTransactionResponseType, @@ -116,8 +117,7 @@ def test_connect_wallet(self): descriptor_patch.assert_has_calls(expected_calls) database_config_memory_patch.assert_called() - block_chain_config_electrum_mock.assert_called_with( - electrum_config_mock) + block_chain_config_electrum_mock.assert_called_with(electrum_config_mock) electrum_config_patch.assert_called_with( wallet_details_mock.electrum_url, None, 2, 30, 100, True ) @@ -188,8 +188,7 @@ def test_connect_wallet_with_wallet_without_change_descriptor(self): descriptor_patch.assert_has_calls(expected_calls) database_config_memory_patch.assert_called() - block_chain_config_electrum_mock.assert_called_with( - electrum_config_mock) + block_chain_config_electrum_mock.assert_called_with(electrum_config_mock) electrum_config_patch.assert_called_with( wallet_details_mock.electrum_url, None, 2, 30, 100, True ) @@ -293,8 +292,7 @@ def test_connect_wallet_with_existing_wallet_but_differing_wallet_id_in_db( descriptor_patch.assert_has_calls(expected_calls) database_config_memory_patch.assert_called() - block_chain_config_electrum_mock.assert_called_with( - electrum_config_mock) + block_chain_config_electrum_mock.assert_called_with(electrum_config_mock) electrum_config_patch.assert_called_with( wallet_details_mock.electrum_url, None, 2, 30, 100, True ) @@ -343,8 +341,7 @@ def test_create_wallet(self): ): mock_wallet = MagicMock() wallet_model_patch.return_value = mock_wallet - wallet_model_patch.get_current_wallet = MagicMock( - return_value=None) + wallet_model_patch.get_current_wallet = MagicMock(return_value=None) add_mock = MagicMock() commit_mock = MagicMock() @@ -384,8 +381,7 @@ def test_create_wallet_with_no_change_descriptor(self): ): mock_wallet = MagicMock() wallet_model_patch.return_value = mock_wallet - wallet_model_patch.get_current_wallet = MagicMock( - return_value=None) + wallet_model_patch.get_current_wallet = MagicMock(return_value=None) add_mock = MagicMock() commit_mock = MagicMock() @@ -425,8 +421,7 @@ def test_create_wallet_with_existing_current_wallet_in_db(self): ): mock_wallet = MagicMock() wallet_model_patch.return_value = mock_wallet - wallet_model_patch.get_current_wallet = MagicMock( - return_value=MagicMock) + wallet_model_patch.get_current_wallet = MagicMock(return_value=MagicMock) add_mock = MagicMock() commit_mock = MagicMock() @@ -594,8 +589,7 @@ def test_build_transaction(self): output_count, ) - amount_in_each_output = ( - local_utxo_mock.txout.value / 2) / output_count + amount_in_each_output = (local_utxo_mock.txout.value / 2) / output_count tx_builder_mock.add_recipient.assert_called_with( script_mock, amount_in_each_output ) @@ -703,10 +697,8 @@ def test_get_fee_estimate_for_utxos(self): assert fee_estimate_response.status == "success" fee: int = cast(int, transaction_details_mock.fee) - expected_fee_percent = ( - fee / (transaction_details_mock.sent + fee)) * 100 - assert fee_estimate_response.data == FeeDetails( - expected_fee_percent, fee) + expected_fee_percent = (fee / (transaction_details_mock.sent + fee)) * 100 + assert fee_estimate_response.data == FeeDetails(expected_fee_percent, fee) assert fee_estimate_response.psbt == "mock_psbt" def test_get_fee_estimate_for_utxo_with_build_tx_unspendable(self): @@ -726,8 +718,7 @@ def test_get_fee_estimate_for_utxo_with_build_tx_unspendable(self): assert get_fee_estimate_response.data == None def test_get_fee_estimate_for_utxo_with_build_tx_error(self): - build_transaction_error_response = BuildTransactionResponseType( - "error", None) + build_transaction_error_response = BuildTransactionResponseType("error", None) with patch.object( WalletService, "build_transaction", @@ -872,8 +863,7 @@ def test_create_spendable_wallet(self): ) database_config_memory_patch.assert_called() - block_chain_config_electrum_mock.assert_called_with( - electrum_config_mock) + block_chain_config_electrum_mock.assert_called_with(electrum_config_mock) electrum_config_patch.assert_called_with( electrum_url_mock, None, 2, 30, 100, True ) @@ -895,8 +885,8 @@ def test_get_all_transactions_success(self): patch( "src.services.wallet.wallet.electrum_request" ) as mock_electrum_request, + patch.object(WalletService, "wallet") as mock_wallet, ): - mock_wallet = MagicMock() # mock the transactions that bdk fetches from the wallet mock_wallet.list_transactions.return_value = [ Mock(txid="txid1"), @@ -994,90 +984,106 @@ def test_get_all_transactions_without_port(self): assert get_all_transactions_response == [] def test_get_all_outputs(self): - mock_wallet = Mock() - self.wallet_service.wallet = mock_wallet - mock_wallet.is_mine = Mock() - # mark first output as mine and the second as not - annominity_set_count_mock = 2 - mock_wallet.is_mine.side_effect = [True, False] - self.wallet_service.get_all_transactions = Mock( - return_value=all_transactions_mock - ) - mock_annominity_sets = { - all_transactions_mock[0].outputs[0].value: annominity_set_count_mock, - all_transactions_mock[0].outputs[1].value: annominity_set_count_mock, - } - self.wallet_service.calculate_output_annominity_sets = Mock( - return_value=mock_annominity_sets - ) - mock_db_output_1 = Mock() - mock_db_output_1.id = 1 - mock_db_output_1.txid = all_transactions_mock[0].txid - mock_db_output_1.vout = all_transactions_mock[0].outputs[0].output_n - mock_db_output_1.labels = [] - - # This db output is ignored because of the is_mine call therefore we don't - # need to bother mocking all the values - mock_db_output_2 = Mock() - - mock_db_output_synced = [mock_db_output_1, mock_db_output_2] - self.wallet_service.sync_local_db_with_incoming_output = Mock( - side_effect=mock_db_output_synced - ) + with ( + patch.object(WalletService, "wallet") as mock_wallet, + patch.object( + WalletService, "get_all_transactions" + ) as mock_get_all_transactions, + patch.object( + WalletService, "calculate_output_annominity_sets" + ) as mock_calculate_output_annominity_sets, + patch.object( + WalletService, "sync_local_db_with_incoming_output" + ) as mock_sync_local_db_with_incoming_output, + patch.object( + LastFetchedService, "update_last_fetched_outputs_type" + ) as mock_update_last_fetched_outputs_type, + ): + mock_wallet.is_mine = Mock() + # mark first output as mine and the second as not + annominity_set_count_mock = 2 + mock_wallet.is_mine.side_effect = [True, False] + mock_get_all_transactions.return_value = all_transactions_mock + mock_annominity_sets = { + all_transactions_mock[0].outputs[0].value: annominity_set_count_mock, + all_transactions_mock[0].outputs[1].value: annominity_set_count_mock, + } + mock_calculate_output_annominity_sets.return_value = mock_annominity_sets + mock_db_output_1 = Mock() + mock_db_output_1.id = 1 + mock_db_output_1.txid = all_transactions_mock[0].txid + mock_db_output_1.vout = all_transactions_mock[0].outputs[0].output_n + mock_db_output_1.labels = [] + + # This db output is ignored because of the is_mine call therefore we don't + # need to bother mocking all the values + mock_db_output_2 = Mock() + + mock_db_output_synced = [mock_db_output_1, mock_db_output_2] + mock_sync_local_db_with_incoming_output.side_effect = mock_db_output_synced - # call the method we are testing - get_all_outputs_response = self.wallet_service.get_all_outputs() + # call the method we are testing + get_all_outputs_response = self.wallet_service.get_all_outputs() - self.wallet_service.get_all_transactions.assert_called() - self.wallet_service.calculate_output_annominity_sets.assert_called() - self.wallet_service.sync_local_db_with_incoming_output.assert_called() + mock_get_all_transactions.assert_called() + mock_calculate_output_annominity_sets.assert_called() + mock_sync_local_db_with_incoming_output.assert_called() + mock_update_last_fetched_outputs_type.assert_called() - # the returned outputs should only be the first one since we mocked out that the second output would return False when checking if it is mine + # the returned outputs should only be the first one since we mocked out that the second output would return False when checking if it is mine - assert mock_wallet.is_mine.call_count == 2 - assert len(get_all_outputs_response) == 1 + assert mock_wallet.is_mine.call_count == 2 + assert len(get_all_outputs_response) == 1 - assert get_all_outputs_response[0].annominity_set == 2 - assert get_all_outputs_response[0].txid == all_transactions_mock[0].txid - assert get_all_outputs_response[0].labels == [] + assert get_all_outputs_response[0].annominity_set == 2 + assert get_all_outputs_response[0].txid == all_transactions_mock[0].txid + assert get_all_outputs_response[0].labels == [] def test_get_all_outputs_if_none_are_mine(self): - mock_wallet = Mock() - self.wallet_service.wallet = mock_wallet - mock_wallet.is_mine = Mock(return_value=False) - # mark No outputs as mine - annominity_set_count_mock = 2 - self.wallet_service.get_all_transactions = Mock( - return_value=all_transactions_mock - ) - mock_annominity_sets = { - all_transactions_mock[0].outputs[0].value: annominity_set_count_mock, - all_transactions_mock[0].outputs[1].value: annominity_set_count_mock, - } - self.wallet_service.calculate_output_annominity_sets = Mock( - return_value=mock_annominity_sets - ) + with ( + patch.object(WalletService, "wallet") as mock_wallet, + patch.object( + WalletService, "get_all_transactions" + ) as mock_get_all_transactions, + patch.object( + WalletService, "calculate_output_annominity_sets" + ) as mock_calculate_output_annominity_sets, + patch.object( + WalletService, "sync_local_db_with_incoming_output" + ) as mock_sync_local_db_with_incoming_output, + patch.object( + LastFetchedService, "update_last_fetched_outputs_type" + ) as mock_update_last_fetched_outputs_type, + ): + mock_wallet.is_mine = Mock(return_value=False) + # mark No outputs as mine + annominity_set_count_mock = 2 + mock_get_all_transactions.return_value = all_transactions_mock - # These db outputs are ignored because of the is_mine call therefore we don't - # need to bother mocking all the values + mock_annominity_sets = { + all_transactions_mock[0].outputs[0].value: annominity_set_count_mock, + all_transactions_mock[0].outputs[1].value: annominity_set_count_mock, + } + mock_calculate_output_annominity_sets.return_value = mock_annominity_sets - mock_db_output_1 = Mock() - mock_db_output_2 = Mock() + # These db outputs are ignored because of the is_mine call therefore we don't + # need to bother mocking all the values + mock_db_output_1 = Mock() + mock_db_output_2 = Mock() - mock_db_output_synced = [mock_db_output_1, mock_db_output_2] - self.wallet_service.sync_local_db_with_incoming_output = Mock( - side_effect=mock_db_output_synced - ) + mock_db_output_synced = [mock_db_output_1, mock_db_output_2] + mock_sync_local_db_with_incoming_output.side_effect = mock_db_output_synced - # call the method we are testing - get_all_outputs_response = self.wallet_service.get_all_outputs() + # call the method we are testing + get_all_outputs_response = self.wallet_service.get_all_outputs() - self.wallet_service.get_all_transactions.assert_called() - self.wallet_service.calculate_output_annominity_sets.assert_called() - self.wallet_service.sync_local_db_with_incoming_output.assert_called() + mock_get_all_transactions.assert_called() + mock_calculate_output_annominity_sets.assert_called() + mock_sync_local_db_with_incoming_output.assert_not_called() + mock_update_last_fetched_outputs_type.assert_not_called() - assert mock_wallet.is_mine.call_count == 2 - assert get_all_outputs_response == [] + assert mock_wallet.is_mine.call_count == 2 + assert get_all_outputs_response == [] def test_calculate_output_annominity_sets(self): first_output_value = tx_mock.outputs[0].value @@ -1136,9 +1142,18 @@ def test_get_output_labels_unique(self): mock_label_two.display_name = "display_two" mock_label_two.description = "description_two" mock_outputlabel = [ - Mock(txid="txid_one", vout=0, labels=[ - mock_label_one, mock_label_two]), - Mock(txid="txid_two", vout=1, labels=[mock_label_one]), + Mock( + txid="txid_one", + vout=0, + labels=[mock_label_one, mock_label_two], + address="mockaddress1", + ), + Mock( + txid="txid_two", + vout=1, + labels=[mock_label_one], + address="mockaddress2", + ), ] mock_query = mock_db_session.query.return_value @@ -1148,73 +1163,105 @@ def test_get_output_labels_unique(self): mock_query.all.return_value = mock_outputlabel response = self.wallet_service.get_output_labels_unique() - print("what r", response) - assert response["txid_one-0"][0].label == mock_label_one.name - assert response["txid_one-0"][0].display_name == mock_label_one.display_name - assert response["txid_one-0"][0].description == mock_label_one.description + print("what we got", response) + assert response["txid_one-0-mockaddress1"][0].label == mock_label_one.name + assert ( + response["txid_one-0-mockaddress1"][0].display_name + == mock_label_one.display_name + ) + assert ( + response["txid_one-0-mockaddress1"][0].description + == mock_label_one.description + ) - assert response["txid_one-0"][1].label == mock_label_two.name - assert response["txid_one-0"][1].display_name == mock_label_two.display_name - assert response["txid_one-0"][1].description == mock_label_two.description + assert response["txid_one-0-mockaddress1"][1].label == mock_label_two.name + assert ( + response["txid_one-0-mockaddress1"][1].display_name + == mock_label_two.display_name + ) + assert ( + response["txid_one-0-mockaddress1"][1].description + == mock_label_two.description + ) - assert response["txid_two-1"][0].label == mock_label_one.name - assert response["txid_two-1"][0].display_name == mock_label_one.display_name - assert response["txid_two-1"][0].description == mock_label_one.description + assert response["txid_two-1-mockaddress2"][0].label == mock_label_one.name + assert ( + response["txid_two-1-mockaddress2"][0].display_name + == mock_label_one.display_name + ) + assert ( + response["txid_two-1-mockaddress2"][0].description + == mock_label_one.description + ) - assert isinstance(response["txid_one-0"][0], OutputLabelDto) - assert isinstance(response["txid_one-0"][1], OutputLabelDto) + assert isinstance(response["txid_one-0-mockaddress1"][0], OutputLabelDto) + assert isinstance(response["txid_one-0-mockaddress1"][1], OutputLabelDto) def test_populate_outputs_and_labels(self): - mock_label_one = dict( - label="label_one", display_name="display_one", description="description_one" - ) + with ( + patch.object( + WalletService, "add_label_to_output" + ) as mock_add_label_to_output, + patch.object( + WalletService, "sync_local_db_with_incoming_output" + ) as mock_sync_local_db_with_incoming_output, + ): + mock_label_one = dict( + label="label_one", + display_name="display_one", + description="description_one", + ) - mock_label_two = dict( - label="label_two", display_name="display_two", description="description_two" - ) + mock_label_two = dict( + label="label_two", + display_name="display_two", + description="description_two", + ) - output_labels_in_populate_format = ( - PopulateOutputLabelsRequestDto.model_validate( - { - "txidone-0": [mock_label_one, mock_label_two], - "txidone-1": [mock_label_one, mock_label_two], - "txidtwo-0": [mock_label_one], - } + output_labels_in_populate_format = ( + PopulateOutputLabelsRequestDto.model_validate( + { + "txidone-0-mockaddress1": [mock_label_one, mock_label_two], + "txidone-1-mockaddress2": [mock_label_one, mock_label_two], + "txidtwo-0-mockaddress3": [mock_label_one], + } + ) ) - ) - mock_add_label_to_output = Mock(return_value=None) - self.wallet_service.add_label_to_output = mock_add_label_to_output - mock_sync_local_db_with_incoming_output = Mock(return_value=None) - self.wallet_service.sync_local_db_with_incoming_output = ( - mock_sync_local_db_with_incoming_output - ) + mock_add_label_to_output.return_value = None + # self.wallet_service.add_label_to_output = mock_add_label_to_output - # call the method we are testing - populate_outputs_and_labels_response = ( - self.wallet_service.populate_outputs_and_labels( - output_labels_in_populate_format + # call the method we are testing + populate_outputs_and_labels_response = ( + self.wallet_service.populate_outputs_and_labels( + output_labels_in_populate_format + ) ) - ) - # called once for each output - assert mock_sync_local_db_with_incoming_output.call_count == 3 - mock_sync_local_db_with_incoming_output.assert_any_call("txidone", 0) - mock_sync_local_db_with_incoming_output.assert_any_call("txidone", 1) + # called once for each output + assert mock_sync_local_db_with_incoming_output.call_count == 3 + mock_sync_local_db_with_incoming_output.assert_any_call( + "txidone", 0, "mockaddress1" + ) + mock_sync_local_db_with_incoming_output.assert_any_call( + "txidone", 1, "mockaddress2" + ) - mock_sync_local_db_with_incoming_output.assert_any_call("txidtwo", 0) + mock_sync_local_db_with_incoming_output.assert_any_call( + "txidtwo", 0, "mockaddress3" + ) - mock_add_label_to_output.assert_any_call( - "txidone", 0, mock_label_one["display_name"] - ) + mock_add_label_to_output.assert_any_call( + "txidone", 0, mock_label_one["display_name"] + ) - mock_add_label_to_output.assert_any_call( - "txidone", 1, mock_label_two["display_name"] - ) - mock_add_label_to_output.assert_any_call( - "txidtwo", 0, mock_label_one["display_name"] - ) + mock_add_label_to_output.assert_any_call( + "txidone", 1, mock_label_two["display_name"] + ) + mock_add_label_to_output.assert_any_call( + "txidtwo", 0, mock_label_one["display_name"] + ) - assert populate_outputs_and_labels_response == None + assert populate_outputs_and_labels_response == None def test_get_utxos_info(self): mock_outpoint_one = Mock() @@ -1234,8 +1281,7 @@ def test_get_utxos_info(self): mock_all_utxos_three = Mock() mock_all_utxos_three.outpoint = Mock() - all_utxos_mock = [mock_all_utxos_one, - mock_all_utxos_two, mock_all_utxos_three] + all_utxos_mock = [mock_all_utxos_one, mock_all_utxos_two, mock_all_utxos_three] mock_get_all_utxos = Mock(return_value=all_utxos_mock) self.wallet_service.get_all_utxos = mock_get_all_utxos @@ -1246,20 +1292,29 @@ def test_get_utxos_info(self): assert response[0] == mock_all_utxos_one def test_sync_local_db_with_incoming_output_for_new_output(self): - with patch("src.services.wallet.wallet.OutputModel") as mock_output_model: + with ( + patch("src.services.wallet.wallet.OutputModel") as mock_output_model, + patch.object(WalletService, "add_output_to_db") as mock_add_output_to_db, + ): mock_new_output_model = Mock() + mock_new_last_fetched_model = Mock() # return None, as if we did not find this OutputModel in the db mock_output_model.query.filter_by.return_value.first.return_value = None - mock_add_output_to_db = Mock(return_value=mock_new_output_model) - - self.wallet_service.add_output_to_db = mock_add_output_to_db + mock_add_output_to_db.return_value = mock_new_output_model + mock_new_last_fetched_model = Mock(return_value=mock_new_last_fetched_model) response = self.wallet_service.sync_local_db_with_incoming_output( - "txid", 0) - mock_add_output_to_db.assert_called_with(txid="txid", vout=0) + "txid", 0, "mock_address" + ) + mock_add_output_to_db.assert_called_with( + txid="txid", vout=0, address="mock_address" + ) assert response == mock_new_output_model def test_sync_local_db_with_incoming_output_for_existing_output(self): - with patch("src.services.wallet.wallet.OutputModel") as mock_output_model: + with ( + patch("src.services.wallet.wallet.OutputModel") as mock_output_model, + patch.object(WalletService, "add_output_to_db") as mock_add_output_to_db, + ): mock_existing_output_model = Mock() # return None, as if we did not find this OutputModel in the db mock_output_model.query.filter_by.return_value.first.return_value = ( @@ -1267,9 +1322,9 @@ def test_sync_local_db_with_incoming_output_for_existing_output(self): ) mock_add_output_to_db = Mock() - self.wallet_service.add_output_to_db = mock_add_output_to_db response = self.wallet_service.sync_local_db_with_incoming_output( - "txid", 0) + "txid", 0, "mock_address" + ) # since the output already exists in the db we should not call add_output_to_db mock_add_output_to_db.assert_not_called() assert response == mock_existing_output_model From 635560fc6338142b1d71f75b2545538d92c5a08b Mon Sep 17 00:00:00 2001 From: Joseph Wyman Date: Tue, 5 Nov 2024 07:06:12 -0500 Subject: [PATCH 31/85] handle None case for refetching outputs --- backend/src/services/privacy_metrics/privacy_metrics.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/backend/src/services/privacy_metrics/privacy_metrics.py b/backend/src/services/privacy_metrics/privacy_metrics.py index 77cf26ed..1cc734c1 100644 --- a/backend/src/services/privacy_metrics/privacy_metrics.py +++ b/backend/src/services/privacy_metrics/privacy_metrics.py @@ -115,7 +115,12 @@ def analyze_no_address_reuse( ) now = datetime.now() refetch_interval = timedelta(minutes=5) - should_refetch_outputs = now - last_fetched_output_datetime > refetch_interval + + should_refetch_outputs = ( + now - last_fetched_output_datetime > refetch_interval + if last_fetched_output_datetime is not None + else True # if last_fetched_output_datetime is None, we should "fetch" for first time + ) if last_fetched_output_datetime is None or should_refetch_outputs: LOGGER.info("No last fetched output datetime found, fetching all outputs") From e1eb6b524379d47e1d0840b3ff642204648ef0a5 Mon Sep 17 00:00:00 2001 From: Joseph Wyman Date: Thu, 7 Nov 2024 08:56:05 -0500 Subject: [PATCH 32/85] add some transaction details to the database, add privacy analysis for no small change, no change, no round payments, add last fetched tracking of transactions --- backend/src/models/last_fetched.py | 1 + backend/src/models/transaction.py | 20 ++++ .../my_types/controller_types/utxos_dtos.py | 2 +- .../last_fetched/last_fetched_service.py | 30 ++++++ .../privacy_metrics/privacy_metrics.py | 78 ++++++++++++++-- backend/src/services/wallet/wallet.py | 93 +++++++++++++++---- 6 files changed, 194 insertions(+), 30 deletions(-) create mode 100644 backend/src/models/transaction.py diff --git a/backend/src/models/last_fetched.py b/backend/src/models/last_fetched.py index 2beec075..0334c903 100644 --- a/backend/src/models/last_fetched.py +++ b/backend/src/models/last_fetched.py @@ -7,6 +7,7 @@ class LastFetchedType(PyEnum): OUTPUTS = "outputs" + TRANSACTIONS = "transactions" class LastFetched(DB.Model): diff --git a/backend/src/models/transaction.py b/backend/src/models/transaction.py new file mode 100644 index 00000000..b30091c9 --- /dev/null +++ b/backend/src/models/transaction.py @@ -0,0 +1,20 @@ +from sqlalchemy import String, Integer + + +from src.database import DB + + +class Transaction(DB.Model): + __tablename__ = "transactions" + + id = DB.Column(Integer, primary_key=True, autoincrement=True) + txid = DB.Column(String, unique=True, nullable=False) + received_amount = DB.Column(Integer, nullable=False) + sent_amount = DB.Column(Integer, nullable=False) + fee = DB.Column(Integer, nullable=False) + + # at some point I will need an output relationship + # prob just need it on the output + # outputs = DB.relationship( + # "Output", secondary=output_labels, back_populates="transactions" + # ) diff --git a/backend/src/my_types/controller_types/utxos_dtos.py b/backend/src/my_types/controller_types/utxos_dtos.py index a8d0539b..af2ede9d 100644 --- a/backend/src/my_types/controller_types/utxos_dtos.py +++ b/backend/src/my_types/controller_types/utxos_dtos.py @@ -65,7 +65,7 @@ class TransactionDetailDto(BaseModel): network: str # bitcoin, what are the other options? witness_type: str # segwit, what are the other options? coinbase: bool - flag: int + flag: Optional[int] txhash: str confirmations: Optional[int] block_height: Optional[int] diff --git a/backend/src/services/last_fetched/last_fetched_service.py b/backend/src/services/last_fetched/last_fetched_service.py index f80df17c..b72b265e 100644 --- a/backend/src/services/last_fetched/last_fetched_service.py +++ b/backend/src/services/last_fetched/last_fetched_service.py @@ -38,3 +38,33 @@ def get_last_fetched_output_datetime( if last_fetched_output: return last_fetched_output.timestamp return None + + @classmethod + def update_last_fetched_transaction_type( + self, + ) -> None: + """Update the last fetched time for the transactions.""" + timestamp = datetime.now() + current_last_fetched_output = LastFetched.query.filter_by( + type=LastFetchedType.TRANSACTIONS + ).first() + if current_last_fetched_output: + current_last_fetched_output.timestamp = timestamp + else: + last_fetched_output = LastFetched( + type=LastFetchedType.TRANSACTIONS, timestamp=timestamp + ) + DB.session.add(last_fetched_output) + DB.session.commit() + + @classmethod + def get_last_fetched_transaction_datetime( + self, + ) -> Optional[datetime]: + """Get the last fetched time for the transactions.""" + last_fetched_output = LastFetched.query.filter_by( + type=LastFetchedType.TRANSACTIONS + ).first() + if last_fetched_output: + return last_fetched_output.timestamp + return None diff --git a/backend/src/services/privacy_metrics/privacy_metrics.py b/backend/src/services/privacy_metrics/privacy_metrics.py index 1cc734c1..7eeb9ac3 100644 --- a/backend/src/services/privacy_metrics/privacy_metrics.py +++ b/backend/src/services/privacy_metrics/privacy_metrics.py @@ -1,8 +1,11 @@ +from typing import Optional +from bdkpython import bdk +from bitcoinlib.transactions import Transaction from src.database import DB from src.models.privacy_metric import PrivacyMetric, PrivacyMetricName from src.models.outputs import Output as OutputModel from datetime import datetime, timedelta - +from src.models.transaction import Transaction as TransactionModel import structlog @@ -19,7 +22,12 @@ def get_all_privacy_metrics(cls) -> list[PrivacyMetric]: def analyze_tx_privacy( cls, txid: str, privacy_metrics: list[PrivacyMetricName] ) -> dict[PrivacyMetricName, bool]: + from src.services.wallet.wallet import WalletService + results: dict[PrivacyMetricName, bool] = dict() + + transaction_details = WalletService.get_transaction_details(txid) + transaction = WalletService.get_transaction(txid) for privacy_metric in privacy_metrics: if privacy_metric == PrivacyMetricName.ANNOMINITY_SET: mock_set = 5 @@ -42,15 +50,15 @@ def analyze_tx_privacy( results[privacy_metric] = result elif privacy_metric == PrivacyMetricName.NO_CHANGE: - result = cls.analyze_no_change(txid) + result = cls.analyze_no_change(transaction_details) results[privacy_metric] = result elif privacy_metric == PrivacyMetricName.NO_SMALL_CHANGE: - result = cls.analyze_no_small_change(txid) + result = cls.analyze_no_small_change(transaction_details) results[privacy_metric] = result elif privacy_metric == PrivacyMetricName.NO_ROUND_NUMBER_PAYMENTS: - result = cls.analyze_no_round_number_payments(txid) + result = cls.analyze_no_round_number_payments(transaction) results[privacy_metric] = result elif privacy_metric == PrivacyMetricName.SAME_SCRIPT_TYPES: @@ -150,16 +158,68 @@ def analyze_timing_analysis(cls, txid: str) -> bool: return True @classmethod - def analyze_no_change(cls, txid: str) -> bool: + def analyze_no_change(cls, transaction_details: Optional[TransactionModel]) -> bool: + if transaction_details is None: + return False + if ( + transaction_details.sent_amount > 0 + and transaction_details.received_amount > 0 + ): + return False return True @classmethod - def analyze_no_small_change(cls, txid: str) -> bool: - return True + def analyze_no_small_change( + cls, transaction_details: Optional[TransactionModel] + ) -> bool: + if transaction_details is None: + return False + + total_change = ( + transaction_details.sent_amount - transaction_details.received_amount + ) + # 50,000 sats is about $35 at $69k + # when fees are at 100 sat/vbyte, this utxo would cost about + # 30% in fees of its total amount. + # TODO should this be 100,000 sats? + acceptable_change_amount = 50000 + if total_change > acceptable_change_amount: + return False + else: + return True @classmethod - def analyze_no_round_number_payments(cls, txid: str) -> bool: - return True + def analyze_no_round_number_payments( + cls, transaction: Optional[Transaction] + ) -> bool: + """Check if this transaction's change is easily detectable due to + having a round number payment output and a non round number change + output. + Use the last four digits to determine if an output is a round number. + This privacy check will fail if there is an output with a last four + digits of 0 and an output with a last four digits not equal to 0. + """ + + if transaction is None: + return False + + is_non_round_output_count = 0 + is_round_output_count = 0 + outputs = transaction.outputs + for output in outputs: + last_four_digits = output.value % 10000 + if last_four_digits == 0: + is_round_output_count += 1 + else: + is_non_round_output_count += 1 + + if is_non_round_output_count > 0 and is_round_output_count > 0: + # there is a round number output and a non round number output + # which means it is easy to detect the change output, + # which is the non round number output. + return False + else: + return True @classmethod def analyze_same_script_types(cls, txid: str) -> bool: diff --git a/backend/src/services/wallet/wallet.py b/backend/src/services/wallet/wallet.py index 5b7929ee..557cf0c4 100644 --- a/backend/src/services/wallet/wallet.py +++ b/backend/src/services/wallet/wallet.py @@ -2,6 +2,7 @@ import bdkpython as bdk from bitcoinlib.transactions import Output, Transaction from sqlalchemy import func +from src.models.transaction import Transaction as TransactionModel from src.models.label import Label from src.models.outputs import Output as OutputModel from typing import Literal, Optional, List, Dict @@ -293,43 +294,95 @@ def get_all_utxos(self) -> List[bdk.LocalUtxo]: def get_all_transactions( cls, ) -> List[Transaction]: - """Get all transactions for the current wallet.""" + """Get all transactions for the current wallet. + + Add the transaction to the database. + Add to the database that the transactions have been fetched + via the LastFetched model. + """ wallet_details = Wallet.get_current_wallet() if cls.wallet is None or wallet_details is None: LOGGER.error("No electrum wallet or wallet details found.") return [] + transactions: list[bdk.TransactionDetails] = cls.wallet.list_transactions(False) + + all_tx_details: List[Transaction] = [] + + for index, transaction in enumerate(transactions): + transaction_response = cls.get_transaction(transaction.txid, index) + if transaction_response is not None: + # add the transaction to the database + # use the bdk transaction details since it contains + # the sent and received amounts relative to the users wallet + # instead of just agnostic values that electrum returns + cls.add_transaction_to_db(transaction) + all_tx_details.append(transaction_response) + else: + LOGGER.error(f"Error getting transaction {transaction.txid}") + + # mark transactions as fetched + LastFetchedService.update_last_fetched_transaction_type() + + return all_tx_details + + @classmethod + def get_transaction(cls, txid, index=1) -> Optional[Transaction]: + "Get an individual transaction from the wallet by the txid." + wallet_details = Wallet.get_current_wallet() + if wallet_details is None: + LOGGER.error( + "No wallet_details found, therefore the request to get the transaction can not be made." + ) + return None + electrum_url = wallet_details.electrum_url if electrum_url is None: LOGGER.error("No electrum url found in the wallet details") - return [] + return None url, port = parse_electrum_url(electrum_url) if url is None or port is None: LOGGER.error("No electrum url or port found in the wallet details") - return [] + return None + + electrum_response = electrum_request( + url, + int(port), + ElectrumMethod.GET_TRANSACTIONS, + GetTransactionsRequestParams(txid, False), + index, + ) - transactions = cls.wallet.list_transactions(False) + if electrum_response.status == "success" and electrum_response.data is not None: + transaction: Transaction = electrum_response.data + return transaction + else: + return None - all_tx_details: List[Transaction] = [] + @classmethod + def get_transaction_details(cls, txid) -> Optional[TransactionModel]: + """Get the transaction details from the database.""" + transaction = DB.session.query(TransactionModel).filter_by(txid=txid).first() + return transaction - for index, transaction in enumerate(transactions): - electrum_response = electrum_request( - url, - int(port), - ElectrumMethod.GET_TRANSACTIONS, - GetTransactionsRequestParams(transaction.txid, False), - index, + @classmethod + def add_transaction_to_db(cls, transaction_details: bdk.TransactionDetails): + existing_transaction = ( + DB.session.query(TransactionModel) + .filter_by(txid=transaction_details.txid) + .first() + ) + if existing_transaction is None: + new_transaction = TransactionModel( + txid=transaction_details.txid, + received_amount=transaction_details.received, + sent_amount=transaction_details.sent, + fee=transaction_details.fee, ) - - if ( - electrum_response.status == "success" - and electrum_response.data is not None - ): - transaction: Transaction = electrum_response.data - all_tx_details.append(electrum_response.data) - return all_tx_details + DB.session.add(new_transaction) + DB.session.commit() @classmethod def get_all_outputs(cls) -> List[LiveWalletOutput]: From ef79eb645e82973d8fa079e61a864b62dfd5fc58 Mon Sep 17 00:00:00 2001 From: Joseph Wyman Date: Fri, 8 Nov 2024 06:03:51 -0500 Subject: [PATCH 33/85] create spends in mock wallet creation, and add same script type privacy check --- backend/src/controllers/wallet.py | 66 +++++++++++++++---- .../my_types/controller_types/utxos_dtos.py | 2 +- .../privacy_metrics/privacy_metrics.py | 45 ++++++++++++- backend/src/services/wallet/wallet.py | 6 +- backend/src/testbridge/ngiri.py | 20 ++++-- backend/src/testbridge/wallet_spends.py | 44 +++++++++++++ 6 files changed, 161 insertions(+), 22 deletions(-) create mode 100644 backend/src/testbridge/wallet_spends.py diff --git a/backend/src/controllers/wallet.py b/backend/src/controllers/wallet.py index 6839d028..77080a0e 100644 --- a/backend/src/controllers/wallet.py +++ b/backend/src/controllers/wallet.py @@ -1,12 +1,13 @@ from typing import Annotated, Optional from bdkpython import bdk +from bitcoinlib.transactions import functools from flask import Blueprint, request from time import sleep from dependency_injector.wiring import inject, Provide import structlog -from src.testbridge.ngiri import randomly_fund_mock_wallet +from src.testbridge.ngiri import mine_a_block_to_miner, randomly_fund_mock_wallet from src.my_types import ScriptType from src.services import WalletService from src.containers.service_container import ServiceContainer @@ -17,6 +18,13 @@ ValidationErrorResponse, SimpleErrorResponse, ) +from src.testbridge.wallet_spends import create_and_broadcast_transaction_for_bdk_wallet + +from src.services.wallet.raw_output_script_examples import ( + p2pkh_raw_output_script, + p2sh_raw_output_script, + p2wsh_raw_output_script, +) wallet_api = Blueprint("wallet", __name__, url_prefix="/wallet") @@ -158,10 +166,11 @@ def get_wallet_type( def create_spendable_wallet(): """ Create a new wallet with spendable UTXOs. + Spend those utxos in a few transactions. + Then give the wallet a few more UTXOs. """ try: - data = CreateSpendableWalletRequestDto.model_validate_json( - request.data) + data = CreateSpendableWalletRequestDto.model_validate_json(request.data) bdk_network: bdk.Network = bdk.Network.__members__[data.network] @@ -176,25 +185,60 @@ def create_spendable_wallet(): if wallet_descriptor is None: return ( - SimpleErrorResponse( - message="Error creating wallet").model_dump(), + SimpleErrorResponse(message="Error creating wallet").model_dump(), 400, ) - wallet = WalletService.create_spendable_wallet( - bdk_network, wallet_descriptor) - # fund wallet + (wallet, blockchain) = WalletService.create_spendable_wallet( + bdk_network, wallet_descriptor + ) try: wallet_address = wallet.get_address(bdk.AddressIndex.LAST_UNUSED()) + # fund the wallet randomly_fund_mock_wallet( wallet_address.address.as_string(), float(data.minUtxoAmount), float(data.maxUtxoAmount), int(data.utxoCount), ) - # need a second for the tx to be mined - sleep(2) - except Exception: + + # make sure a few blocks are mined before continuing + # to ensure the wallet is funded. + mine_a_block_to_miner() + mine_a_block_to_miner() + mine_a_block_to_miner() + + # sync the wallet so the wallet knows about latest transactions + wallet.sync(blockchain, None) + + # create and broadcast a handful of transactions + create_and_broadcast_transaction_for_bdk_wallet( + wallet, blockchain, 50000, 10, p2sh_raw_output_script + ) + mine_a_block_to_miner() + wallet.sync(blockchain, None) + create_and_broadcast_transaction_for_bdk_wallet( + wallet, blockchain, 50000, 10, p2pkh_raw_output_script + ) + mine_a_block_to_miner() + wallet.sync(blockchain, None) + create_and_broadcast_transaction_for_bdk_wallet( + wallet, blockchain, 50000, 10, p2wsh_raw_output_script + ) + + # Fund the wallet again so that there are a bunch of utxos + # instead of just one because the spends are spend alls. + mine_a_block_to_miner() + wallet.sync(blockchain, None) + randomly_fund_mock_wallet( + wallet_address.address.as_string(), + float(data.minUtxoAmount), + float(data.maxUtxoAmount), + int(data.utxoCount), + ) + + except Exception as e: + LOGGER.error("error funding wallet", error=e) return SimpleErrorResponse(message="Error funding wallet").model_dump(), 400 return CreateSpendableWalletResponseDto( diff --git a/backend/src/my_types/controller_types/utxos_dtos.py b/backend/src/my_types/controller_types/utxos_dtos.py index af2ede9d..eeeaf8a2 100644 --- a/backend/src/my_types/controller_types/utxos_dtos.py +++ b/backend/src/my_types/controller_types/utxos_dtos.py @@ -17,7 +17,7 @@ class InputDto(BaseModel): script_type: str # sig_pubkey address: str value: int - public_keys: str + public_keys: Optional[str | List[str]] compressed: bool compressed: bool encoding: str # bech32, what other options? diff --git a/backend/src/services/privacy_metrics/privacy_metrics.py b/backend/src/services/privacy_metrics/privacy_metrics.py index 7eeb9ac3..985f99cf 100644 --- a/backend/src/services/privacy_metrics/privacy_metrics.py +++ b/backend/src/services/privacy_metrics/privacy_metrics.py @@ -62,7 +62,9 @@ def analyze_tx_privacy( results[privacy_metric] = result elif privacy_metric == PrivacyMetricName.SAME_SCRIPT_TYPES: - result = cls.analyze_same_script_types(txid) + result = cls.analyze_same_script_types( + transaction_details=transaction_details, transaction=transaction + ) results[privacy_metric] = result elif privacy_metric == PrivacyMetricName.AVOID_OUTPUT_SIZE_DIFFERENCE: @@ -222,8 +224,45 @@ def analyze_no_round_number_payments( return True @classmethod - def analyze_same_script_types(cls, txid: str) -> bool: - return True + def analyze_same_script_types( + cls, + transaction_details: Optional[TransactionModel], + transaction: Optional[Transaction], + ) -> bool: + """Analyze if the transaction is a spend to an output with a different + script type than the user's input. + + This would be bad for privacy by linking the user's input to the output + with the same script type. + """ + if transaction is None or transaction_details is None: + return False + + # we are trying to obfuscate the change output + # so if there is no change then this passes + is_no_change = cls.analyze_no_change(transaction_details) + if is_no_change: + return True + + is_the_user_sending_funds = transaction_details.sent_amount > 0 + if is_the_user_sending_funds is False: + # this privacy metric does not apply if the user + # is not sending funds + return True + + output_script_types = set() + + for output in transaction.outputs: + output_script_types.add(output.script_type) + + if len(output_script_types) > 1: + # there are two different output script types + # and we know one must be this users + # linking an input and an output. + # Therefore this privacy metric fails + return False + else: + return True @classmethod def analyze_avoid_output_size_difference(cls, txid: str) -> bool: diff --git a/backend/src/services/wallet/wallet.py b/backend/src/services/wallet/wallet.py index 557cf0c4..190fd931 100644 --- a/backend/src/services/wallet/wallet.py +++ b/backend/src/services/wallet/wallet.py @@ -5,7 +5,7 @@ from src.models.transaction import Transaction as TransactionModel from src.models.label import Label from src.models.outputs import Output as OutputModel -from typing import Literal, Optional, List, Dict +from typing import Literal, Optional, List, Dict, Tuple from src.api import electrum_request, parse_electrum_url from src.api.electrum import ( @@ -202,7 +202,7 @@ def create_spendable_wallet( wallet_descriptor: bdk.Descriptor, # TODO get this url from a config file electrum_url="127.0.0.1:50000", - ) -> bdk.Wallet: + ) -> Tuple[bdk.Wallet, bdk.Blockchain]: """Create a new wallet and sync it to the electrum server.""" db_config = bdk.DatabaseConfig.MEMORY() @@ -221,7 +221,7 @@ def create_spendable_wallet( wallet.sync(blockchain, None) - return wallet + return (wallet, blockchain) @classmethod def create_spendable_descriptor( diff --git a/backend/src/testbridge/ngiri.py b/backend/src/testbridge/ngiri.py index 36f3fbce..e8f35ebb 100644 --- a/backend/src/testbridge/ngiri.py +++ b/backend/src/testbridge/ngiri.py @@ -4,6 +4,8 @@ import asyncio import random +from time import sleep + LOGGER = structlog.get_logger() @@ -27,10 +29,10 @@ async def fund_wallet(address: str, amount: float): data = FundWalletRequestBody(address, amount) async with aiohttp.ClientSession() as session: async with session.post(url, json=asdict(data)) as response: - return await response.text() + response = await response.text() + return response except Exception as e: - LOGGER.error("Failed to fund wallet", - address=address, amount=amount, error=e) + LOGGER.error("Failed to fund wallet", address=address, amount=amount, error=e) async def fund_wallet_with_multiple_txs( @@ -42,7 +44,10 @@ async def fund_wallet_with_multiple_txs( def randomly_fund_mock_wallet( - address: str, amount_min: float, amount_max: float, transaction_count: int + address: str, + amount_min: float, + amount_max: float, + transaction_count: int, ): """Generate transactions inside a min and max btc amount to a specified address""" LOGGER.info( @@ -59,3 +64,10 @@ def randomly_fund_mock_wallet( for response in responses: LOGGER.info(f"Fund wallet response: {response}") + + +def mine_a_block_to_miner(): + mock_miner_mock_address = "n4qq7z7NUXeDW2hjJ4WzwCu8KqVmCXMNEg" + asyncio.run(fund_wallet(mock_miner_mock_address, 0.0001)) + # brief sleep to allow the block to be mined + sleep(2) diff --git a/backend/src/testbridge/wallet_spends.py b/backend/src/testbridge/wallet_spends.py new file mode 100644 index 00000000..089b9720 --- /dev/null +++ b/backend/src/testbridge/wallet_spends.py @@ -0,0 +1,44 @@ +import bdkpython as bdk +from src.services.wallet.raw_output_script_examples import ( + p2sh_raw_output_script, +) + +import structlog + + +LOGGER = structlog.get_logger() + + +def create_and_broadcast_transaction_for_bdk_wallet( + wallet: bdk.Wallet, + blockchain: bdk.Blockchain, + amount: int = 50000, + sats_per_vbyte: int = 10, + raw_output_script: str = p2sh_raw_output_script, +): + "Create, sign and broadcast a transaction for the given bdk wallet" + tx_builder = bdk.TxBuilder() + + utxos = wallet.list_unspent() + # I have no idea how to not select the utxos manually + # therefore just use all the utxos for now. + outpoints = [utxo.outpoint for utxo in utxos] + tx_builder = tx_builder.add_utxos(outpoints) + + tx_builder = tx_builder.fee_rate(sats_per_vbyte) + binary_script = bytes.fromhex(raw_output_script) + + script = bdk.Script(binary_script) + + tx_builder = tx_builder.add_recipient(script, amount) + + built_transaction: bdk.TxBuilderResult = tx_builder.finish(wallet) + signed = wallet.sign(built_transaction.psbt, sign_options=None) + if signed: + transaction = built_transaction.psbt.extract_tx() + LOGGER.info(f"broadcasting {built_transaction.transaction_details}") + blockchain.broadcast(transaction) + else: + LOGGER.error( + "Failed to sign the transaction, therefore it can not be broadcast" + ) From d45432ab30770292ec9b8b3d1b4fc4abaf2e656f Mon Sep 17 00:00:00 2001 From: Joseph Wyman Date: Fri, 8 Nov 2024 09:18:29 -0500 Subject: [PATCH 34/85] add ability to create coinjoin like tx for a wallet, add method to analyze annominity set of a tx for a user --- backend/src/controllers/wallet.py | 19 ++- .../privacy_metrics/privacy_metrics.py | 112 ++++++++++++++---- backend/src/services/wallet/wallet.py | 7 ++ backend/src/testbridge/wallet_spends.py | 61 ++++++++++ 4 files changed, 173 insertions(+), 26 deletions(-) diff --git a/backend/src/controllers/wallet.py b/backend/src/controllers/wallet.py index 77080a0e..1ed1e0bc 100644 --- a/backend/src/controllers/wallet.py +++ b/backend/src/controllers/wallet.py @@ -18,7 +18,10 @@ ValidationErrorResponse, SimpleErrorResponse, ) -from src.testbridge.wallet_spends import create_and_broadcast_transaction_for_bdk_wallet +from src.testbridge.wallet_spends import ( + create_and_broadcast_transaction_for_bdk_wallet, + create_and_broadcast_coinjoin_for_bdk_wallet, +) from src.services.wallet.raw_output_script_examples import ( p2pkh_raw_output_script, @@ -170,7 +173,8 @@ def create_spendable_wallet(): Then give the wallet a few more UTXOs. """ try: - data = CreateSpendableWalletRequestDto.model_validate_json(request.data) + data = CreateSpendableWalletRequestDto.model_validate_json( + request.data) bdk_network: bdk.Network = bdk.Network.__members__[data.network] @@ -185,7 +189,8 @@ def create_spendable_wallet(): if wallet_descriptor is None: return ( - SimpleErrorResponse(message="Error creating wallet").model_dump(), + SimpleErrorResponse( + message="Error creating wallet").model_dump(), 400, ) @@ -222,8 +227,12 @@ def create_spendable_wallet(): ) mine_a_block_to_miner() wallet.sync(blockchain, None) - create_and_broadcast_transaction_for_bdk_wallet( - wallet, blockchain, 50000, 10, p2wsh_raw_output_script + create_and_broadcast_coinjoin_for_bdk_wallet( + wallet, + blockchain, + 50000, + 10, + [p2wsh_raw_output_script, p2pkh_raw_output_script], ) # Fund the wallet again so that there are a bunch of utxos diff --git a/backend/src/services/privacy_metrics/privacy_metrics.py b/backend/src/services/privacy_metrics/privacy_metrics.py index 985f99cf..304238d5 100644 --- a/backend/src/services/privacy_metrics/privacy_metrics.py +++ b/backend/src/services/privacy_metrics/privacy_metrics.py @@ -9,6 +9,8 @@ import structlog +from src.services.wallet.wallet import WalletService + LOGGER = structlog.get_logger() @@ -30,8 +32,8 @@ def analyze_tx_privacy( transaction = WalletService.get_transaction(txid) for privacy_metric in privacy_metrics: if privacy_metric == PrivacyMetricName.ANNOMINITY_SET: - mock_set = 5 - result = cls.analyze_annominit_set(txid, mock_set) + mock_set = 2 + result = cls.analyze_annominity_set(transaction, mock_set) results[privacy_metric] = result elif privacy_metric == PrivacyMetricName.NO_ADDRESS_REUSE: @@ -106,8 +108,69 @@ def analyze_tx_privacy( return results @classmethod - def analyze_annominit_set(cls, txid: str, desired_annominity_set: int) -> bool: - return True + def analyze_annominity_set( + cls, + transaction: Optional[Transaction], + desired_annominity_set: int = 2, + allow_some_uneven_change: bool = True, + ) -> bool: + """Analyze if the users output/s in the tx have the desired annominity + set. + + If allow_some_uneven_change is True, then the privacy metric will pass + if atleast one of the user's outputs is above the + desired annominity set. + + If allow_some_uneven_change is False, then the privacy metric will + only pass if all of the user's outputs are above the desired + annominity set. + """ + if transaction is None: + return False + # compare the users utxos to other utxos, + # if other utxos do not have the same value + # then this fails the annominity set metric + + # Check that all outputs have already been fetched recently + cls.ensure_recently_fetched_outputs() + + output_annominity_count = WalletService.calculate_output_annominity_sets( + transaction.outputs + ) + users_utxos_that_passed_annominity_test = 0 + users_utxos_that_failed_annominity_test = 0 + for output in transaction.outputs: + user_output = WalletService.get_output_from_db( + transaction.txid, output.output_n + ) + is_users_output = user_output is not None + if is_users_output: + annominity_set = output_annominity_count[output.value] + if annominity_set < desired_annominity_set: + # the users output has a lower annominity set + # than the desired annominity set + # therefore this metric fails + users_utxos_that_failed_annominity_test += 1 + else: + users_utxos_that_passed_annominity_test += 1 + + if users_utxos_that_passed_annominity_test == 0: + return False + + if allow_some_uneven_change is False: + # if the user has any utxos that failed the annominity test + # then this metric fails + if users_utxos_that_failed_annominity_test > 0: + return False + else: + # all utxos passed the annominity test + return True + else: # allow_some_uneven_change is True + # at this point we know that the user has at least one utxo + # that passed the annominity test, if the user has other utxos + # that do not have an annominity set above 0 we just consider it part of the + # uneven change that is allowed. + return True @classmethod def analyze_no_address_reuse( @@ -117,25 +180,9 @@ def analyze_no_address_reuse( # can I inject this in? # circular imports are currently preventing it. from src.services.wallet.wallet import WalletService - from src.services.last_fetched.last_fetched_service import LastFetchedService # Check that all outputs have already been fetched recently - last_fetched_output_datetime = ( - LastFetchedService.get_last_fetched_output_datetime() - ) - now = datetime.now() - refetch_interval = timedelta(minutes=5) - - should_refetch_outputs = ( - now - last_fetched_output_datetime > refetch_interval - if last_fetched_output_datetime is not None - else True # if last_fetched_output_datetime is None, we should "fetch" for first time - ) - - if last_fetched_output_datetime is None or should_refetch_outputs: - LOGGER.info("No last fetched output datetime found, fetching all outputs") - # this will get all the outputs and add them to the database, ensuring that they exist - WalletService.get_all_outputs() + cls.ensure_recently_fetched_outputs() outputs = OutputModel.query.filter_by(txid=txid).all() for output in outputs: if WalletService.is_address_reused(output.address): @@ -299,3 +346,26 @@ def analyze_no_post_mix_change(cls, txid: str) -> bool: @classmethod def analyze_segregate_postmix_and_nonmix(cls, txid: str) -> bool: return True + + @classmethod + def ensure_recently_fetched_outputs(cls) -> None: + from src.services.last_fetched.last_fetched_service import LastFetchedService + + # Check that all outputs have already been fetched recently + last_fetched_output_datetime = ( + LastFetchedService.get_last_fetched_output_datetime() + ) + now = datetime.now() + refetch_interval = timedelta(minutes=5) + + should_refetch_outputs = ( + now - last_fetched_output_datetime > refetch_interval + if last_fetched_output_datetime is not None + else True # if last_fetched_output_datetime is None, we should "fetch" for first time + ) + + if last_fetched_output_datetime is None or should_refetch_outputs: + LOGGER.info( + "No last fetched output datetime found, fetching all outputs") + # this will get all the outputs and add them to the database, ensuring that they exist + WalletService.get_all_outputs() diff --git a/backend/src/services/wallet/wallet.py b/backend/src/services/wallet/wallet.py index 190fd931..d90ffbaf 100644 --- a/backend/src/services/wallet/wallet.py +++ b/backend/src/services/wallet/wallet.py @@ -533,6 +533,13 @@ def populate_outputs_and_labels( LOGGER.error("Error populating outputs and labels", error=e) DB.session.rollback() + @classmethod + def get_output_from_db( + cls, txid: str, vout: int + ) -> Optional[OutputModel]: + """Get an output from the database by the txid and vout.""" + output = OutputModel.query.filter_by(txid=txid, vout=vout).first() + return output # TODO should this even go here or in its own service? @classmethod def add_label_to_output( diff --git a/backend/src/testbridge/wallet_spends.py b/backend/src/testbridge/wallet_spends.py index 089b9720..b93105f0 100644 --- a/backend/src/testbridge/wallet_spends.py +++ b/backend/src/testbridge/wallet_spends.py @@ -42,3 +42,64 @@ def create_and_broadcast_transaction_for_bdk_wallet( LOGGER.error( "Failed to sign the transaction, therefore it can not be broadcast" ) + + +def create_and_broadcast_coinjoin_for_bdk_wallet( + wallet: bdk.Wallet, + blockchain: bdk.Blockchain, + amount: int = 50000, + sats_per_vbyte: int = 10, + raw_output_scripts: list[str] = [p2sh_raw_output_script], +): + """Create, sign and broadcast a coinjoin like transaction for the given bdk wallet + + Send an equal amount to each of the given raw_output_scripts which will represent + wallets other than the user's wallet. + + Send an equal amount to the user's wallet 3 times (hard coded below) + TODO make it so you can dynamically specificy how many equal outputs the user should have. + + FYI there will also be a change output that is not of equal value to a change address + for the user's wallet. + """ + wallets_address_1 = wallet.get_address(bdk.AddressIndex.LAST_UNUSED()) + + user_wallet_script_1: bdk.Payload = wallets_address_1.address.script_pubkey() + + wallets_address_2 = wallet.get_address(bdk.AddressIndex.LAST_UNUSED()) + + user_wallet_script_2: bdk.Payload = wallets_address_2.address.script_pubkey() + + wallets_address_3 = wallet.get_address(bdk.AddressIndex.LAST_UNUSED()) + + user_wallet_script_3: bdk.Payload = wallets_address_3.address.script_pubkey() + + tx_builder = bdk.TxBuilder() + + utxos = wallet.list_unspent() + # I have no idea how to not select the utxos manually + # therefore just use all the utxos for now. + outpoints = [utxo.outpoint for utxo in utxos] + tx_builder = tx_builder.add_utxos(outpoints) + + tx_builder = tx_builder.fee_rate(sats_per_vbyte) + + for raw_output_script in raw_output_scripts: + binary_script = bytes.fromhex(raw_output_script) + script = bdk.Script(binary_script) + tx_builder = tx_builder.add_recipient(script, amount) + + tx_builder = tx_builder.add_recipient(user_wallet_script_1, amount) + tx_builder = tx_builder.add_recipient(user_wallet_script_2, amount) + tx_builder = tx_builder.add_recipient(user_wallet_script_3, amount) + + built_transaction: bdk.TxBuilderResult = tx_builder.finish(wallet) + signed = wallet.sign(built_transaction.psbt, sign_options=None) + if signed: + transaction = built_transaction.psbt.extract_tx() + LOGGER.info(f"broadcasting {built_transaction.transaction_details}") + blockchain.broadcast(transaction) + else: + LOGGER.error( + "Failed to sign the transaction, therefore it can not be broadcast" + ) From eb5755add10af256c2b3955d030f3d81f7b0fa66 Mon Sep 17 00:00:00 2001 From: Joseph Wyman Date: Sat, 9 Nov 2024 10:12:55 -0500 Subject: [PATCH 35/85] add analyze no kyc privacy metric --- .../privacy_metrics/privacy_metrics.py | 66 +++++++++++++++++-- 1 file changed, 59 insertions(+), 7 deletions(-) diff --git a/backend/src/services/privacy_metrics/privacy_metrics.py b/backend/src/services/privacy_metrics/privacy_metrics.py index 304238d5..4cc9be0b 100644 --- a/backend/src/services/privacy_metrics/privacy_metrics.py +++ b/backend/src/services/privacy_metrics/privacy_metrics.py @@ -2,6 +2,7 @@ from bdkpython import bdk from bitcoinlib.transactions import Transaction from src.database import DB +from src.models.label import LabelName from src.models.privacy_metric import PrivacyMetric, PrivacyMetricName from src.models.outputs import Output as OutputModel from datetime import datetime, timedelta @@ -30,6 +31,9 @@ def analyze_tx_privacy( transaction_details = WalletService.get_transaction_details(txid) transaction = WalletService.get_transaction(txid) + # Check that all outputs have already been fetched recently + cls.ensure_recently_fetched_outputs() + for privacy_metric in privacy_metrics: if privacy_metric == PrivacyMetricName.ANNOMINITY_SET: mock_set = 2 @@ -90,7 +94,7 @@ def analyze_tx_privacy( results[privacy_metric] = result elif privacy_metric == PrivacyMetricName.NO_KYCED_UTXOS: - result = cls.analyze_no_kyced_utxos(txid) + result = cls.analyze_no_kyced_inputs(transaction=transaction) results[privacy_metric] = result elif privacy_metric == PrivacyMetricName.NO_DUST_ATTACK_UTXOS: @@ -131,9 +135,6 @@ def analyze_annominity_set( # if other utxos do not have the same value # then this fails the annominity set metric - # Check that all outputs have already been fetched recently - cls.ensure_recently_fetched_outputs() - output_annominity_count = WalletService.calculate_output_annominity_sets( transaction.outputs ) @@ -181,8 +182,6 @@ def analyze_no_address_reuse( # circular imports are currently preventing it. from src.services.wallet.wallet import WalletService - # Check that all outputs have already been fetched recently - cls.ensure_recently_fetched_outputs() outputs = OutputModel.query.filter_by(txid=txid).all() for output in outputs: if WalletService.is_address_reused(output.address): @@ -196,10 +195,27 @@ def analyze_minimal_wealth_reveal( cls, txid: str, ) -> bool: + # hmm should this analyze every possibility to see if the user could have revealed a few less sats or + # should it just try to minimize the wealth reveal by a certain amount? like don't reveal more than 100% of the tx toal? + # ... + # get all utxos ever. + # get the ones that were in a transaction before this transaction in question. + # get how much change was + # then check if return True @classmethod def analyze_minimal_tx_history_reveal(cls, txid: str) -> bool: + # if only one utxo was used by this user in this tx then this automaticcaly passes since you cant use less than one utxo in a tx. + # get all utxos ever for this user. + # get only the ones that were created before the transaction in question. + # get the user's utxos that were used in this transaction. and calculate how much total they contributed to the transaction + # then use an algorithm that tries to get the same amount of value out of the users utxos at the time using less utxos than the ones the user used. + # this algo will order all the utxos by size, then it will start by picking the first one, if that is bigger than the total then the metric fails. + # then if there are more than two of the users utxos in the original tx it will combine the top two utxos and if that is more than the utxo total contributed by the user then the metric will fail. + # this will continue until it reachs the point where either the amount of utxos contributed has been tried by the top amount utxos resulting in a passing metric, or less than the amount of utxos contributed as been tried and has found that the user could have used less utxos to create the tx. Thus failing because they linked more utxos together than needed. + # needed as inputs + return True @classmethod @@ -313,14 +329,20 @@ def analyze_same_script_types( @classmethod def analyze_avoid_output_size_difference(cls, txid: str) -> bool: + # is this the same as annominity set or just looser? like the outputs shouldn't be more than 100% different or else + # it is easy to tell the change output? return True @classmethod def analyze_no_unnecessary_input(cls, txid: str) -> bool: + # hmmm how should I do this? return True @classmethod def analyze_use_multi_change_outputs(cls, txid: str) -> bool: + # this hsould be easy + # if you are making a tx (aka include an input) and have a change output + # you should have more than one output. return True @classmethod @@ -329,12 +351,42 @@ def analyze_avoid_common_change_position(cls, txid: str) -> bool: @classmethod def analyze_no_do_not_spend_utxos(cls, txid: str) -> bool: + # this should be easy, get the utxos used, then see if they include the + # label do not spend, if they do this metric fails, if they don't this metric passes. return True @classmethod - def analyze_no_kyced_utxos(cls, txid: str) -> bool: + def analyze_no_kyced_inputs(cls, transaction: Optional[Transaction]) -> bool: + """Check that no inputs in this transaction come from outputs that were marked as being kyced. + + If an input is used that was a utxo marked as kyced then this metric fails. + If no inputs were utxos marked as kyced then this metric passes. + """ + # this should be easy, get the utxos used, then see if they include the + # label kyc, if they do this metric fails, if they don't this metric passes. + + if transaction is None: + return False + + for input in transaction.inputs: + input = input.as_dict() + # get output in the db that each input is refering to. + users_output = WalletService.get_output_from_db( + input["prev_txid"], input["output_n"] + ) + + if users_output is None: + # this is not the users output + # therefore it can not be labeled by the user + # therefore check the next input/output + continue + + if LabelName.KYCED in [label.name for label in users_output.labels]: + return False return True + # I don't like this option any more, dust is more about fees + # a user should just mark a "tracker" output sent to them as do not spend @classmethod def analyze_no_dust_attack_utxos(cls, txid: str) -> bool: return True From 921fe0f442527ccc0518d65156da82e1bccd4de4 Mon Sep 17 00:00:00 2001 From: Joseph Wyman Date: Sat, 9 Nov 2024 10:20:31 -0500 Subject: [PATCH 36/85] add analyze no do not spends privacy metric --- .../privacy_metrics/privacy_metrics.py | 33 +++++++++++++++---- 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/backend/src/services/privacy_metrics/privacy_metrics.py b/backend/src/services/privacy_metrics/privacy_metrics.py index 4cc9be0b..da521cb7 100644 --- a/backend/src/services/privacy_metrics/privacy_metrics.py +++ b/backend/src/services/privacy_metrics/privacy_metrics.py @@ -90,7 +90,8 @@ def analyze_tx_privacy( results[privacy_metric] = result elif privacy_metric == PrivacyMetricName.NO_DO_NOT_SPEND_UTXOS: - result = cls.analyze_no_do_not_spend_utxos(txid) + result = cls.analyze_no_do_not_spend_utxos( + transaction=transaction) results[privacy_metric] = result elif privacy_metric == PrivacyMetricName.NO_KYCED_UTXOS: @@ -350,9 +351,31 @@ def analyze_avoid_common_change_position(cls, txid: str) -> bool: return True @classmethod - def analyze_no_do_not_spend_utxos(cls, txid: str) -> bool: - # this should be easy, get the utxos used, then see if they include the - # label do not spend, if they do this metric fails, if they don't this metric passes. + def analyze_no_do_not_spend_utxos(cls, transaction: Optional[Transaction]) -> bool: + """Check that no inputs in this transaction come from outputs that were marked as do not spend + + If an input is used that was a utxo marked as do not spend then this metric fails. + If no inputs were utxos marked as do not spend then this metric passes. + """ + + if transaction is None: + return False + + for input in transaction.inputs: + input = input.as_dict() + # get output in the db that each input is refering to. + users_output = WalletService.get_output_from_db( + input["prev_txid"], input["output_n"] + ) + + if users_output is None: + # this is not the users output + # therefore it can not be labeled by the user + # therefore check the next input/output + continue + + if LabelName.DO_NOT_SPEND in [label.name for label in users_output.labels]: + return False return True @classmethod @@ -362,8 +385,6 @@ def analyze_no_kyced_inputs(cls, transaction: Optional[Transaction]) -> bool: If an input is used that was a utxo marked as kyced then this metric fails. If no inputs were utxos marked as kyced then this metric passes. """ - # this should be easy, get the utxos used, then see if they include the - # label kyc, if they do this metric fails, if they don't this metric passes. if transaction is None: return False From df33397e587eac9c49078c579e8a6f5bed8d6c7e Mon Sep 17 00:00:00 2001 From: Joseph Wyman Date: Sat, 9 Nov 2024 11:38:09 -0500 Subject: [PATCH 37/85] add method for checking that there is more than one change output for the user --- .../privacy_metrics/privacy_metrics.py | 47 ++++++++++++++++--- backend/src/services/wallet/wallet.py | 18 ++++++- 2 files changed, 58 insertions(+), 7 deletions(-) diff --git a/backend/src/services/privacy_metrics/privacy_metrics.py b/backend/src/services/privacy_metrics/privacy_metrics.py index da521cb7..c1af3406 100644 --- a/backend/src/services/privacy_metrics/privacy_metrics.py +++ b/backend/src/services/privacy_metrics/privacy_metrics.py @@ -82,7 +82,9 @@ def analyze_tx_privacy( results[privacy_metric] = result elif privacy_metric == PrivacyMetricName.USE_MULTI_CHANGE_OUTPUTS: - result = cls.analyze_use_multi_change_outputs(txid) + result = cls.analyze_use_multi_change_outputs( + transaction_details=transaction_details + ) results[privacy_metric] = result elif privacy_metric == PrivacyMetricName.AVOID_COMMON_CHANGE_POSITION: @@ -340,11 +342,44 @@ def analyze_no_unnecessary_input(cls, txid: str) -> bool: return True @classmethod - def analyze_use_multi_change_outputs(cls, txid: str) -> bool: - # this hsould be easy - # if you are making a tx (aka include an input) and have a change output - # you should have more than one output. - return True + def analyze_use_multi_change_outputs( + cls, + transaction_details: Optional[TransactionModel], + ) -> bool: + """Check that the transaction the user made includes more than one 'change' output + to the user. + + If there is no change outputs to the user this metric passes because it does not + actually leak privacy since there is not one easily traceable change output. + If the user has only one change output this metric fails. + If the user has more than one output this metric passes. + + """ + if transaction_details is None: + return False + + total_change = ( + transaction_details.sent_amount - transaction_details.received_amount + ) + + if total_change == 0: + # the user did not receive any change, therefore + # this metric doesn't really apply + # so just return that the metric passes + return True + + # now get how many outputs the user has in this tx + users_outputs = WalletService.get_transaction_outputs_from_db( + transaction_details.txid + ) + if len(users_outputs) == 1: + # there is only one change output for this user + # therefore this metric fails + return False + else: + # the user has more than one change output in the tx + # therefore this metric passes + return True @classmethod def analyze_avoid_common_change_position(cls, txid: str) -> bool: diff --git a/backend/src/services/wallet/wallet.py b/backend/src/services/wallet/wallet.py index d90ffbaf..8913ff8c 100644 --- a/backend/src/services/wallet/wallet.py +++ b/backend/src/services/wallet/wallet.py @@ -535,11 +535,27 @@ def populate_outputs_and_labels( @classmethod def get_output_from_db( - cls, txid: str, vout: int + cls, + txid: str, + vout: int, ) -> Optional[OutputModel]: """Get an output from the database by the txid and vout.""" output = OutputModel.query.filter_by(txid=txid, vout=vout).first() + return output + + @classmethod + def get_transaction_outputs_from_db( + cls, + txid: str, + ) -> List[OutputModel]: + """Get all the wallet's outputs in the db associated with a txid""" + outputs = OutputModel.query.filter_by( + txid=txid, + ).all() + + return outputs + # TODO should this even go here or in its own service? @classmethod def add_label_to_output( From baf31ae83cae671253255a8c043de000cc2591f3 Mon Sep 17 00:00:00 2001 From: Joseph Wyman Date: Sun, 10 Nov 2024 11:10:54 -0500 Subject: [PATCH 38/85] add privacy algo for seeing if least amount of history was revealed --- backend/src/models/outputs.py | 30 +++++- backend/src/models/transaction.py | 22 +++++ .../privacy_metrics/privacy_metrics.py | 72 ++++++++++++-- backend/src/services/wallet/wallet.py | 93 ++++++++++++++++++- 4 files changed, 203 insertions(+), 14 deletions(-) diff --git a/backend/src/models/outputs.py b/backend/src/models/outputs.py index d467b061..05d67468 100644 --- a/backend/src/models/outputs.py +++ b/backend/src/models/outputs.py @@ -10,14 +10,38 @@ class Output(DB.Model): # Auto-incrementing integer id = DB.Column(Integer, primary_key=True, autoincrement=True) - txid = DB.Column( - DB.String(), - nullable=False, + # Foreign key to the 'Transaction' model (creating transaction) + txid = DB.Column(DB.String(), DB.ForeignKey( + "transactions.txid"), nullable=False) + + # Relationship to the 'Transaction' model for the creating transaction + transaction = DB.relationship( + "Transaction", + back_populates="outputs", + # Explicitly tell SQLAlchemy which foreign key to use + foreign_keys=[txid], + ) + + # Foreign key to the 'Transaction' model (spending transaction) + spent_txid = DB.Column( + DB.String(), DB.ForeignKey("transactions.txid"), nullable=True + ) + + # Relationship to the 'Transaction' model for the spending transaction + spent_transaction = DB.relationship( + "Transaction", + back_populates="inputs", + foreign_keys=[ + spent_txid + ], # Explicitly tell SQLAlchemy which foreign key to use ) vout = DB.Column(DB.Integer, nullable=False, default=0) address = DB.Column(DB.String(), nullable=False) + # nullable to make populating outputs from output label population easier + value = DB.Column(DB.Integer, nullable=True) # in sats + # Relationship to labels labels = DB.relationship( "Label", secondary=output_labels, back_populates="outputs") diff --git a/backend/src/models/transaction.py b/backend/src/models/transaction.py index b30091c9..2f7c0b82 100644 --- a/backend/src/models/transaction.py +++ b/backend/src/models/transaction.py @@ -2,6 +2,7 @@ from src.database import DB +from src.models.outputs import Output class Transaction(DB.Model): @@ -12,6 +13,27 @@ class Transaction(DB.Model): received_amount = DB.Column(Integer, nullable=False) sent_amount = DB.Column(Integer, nullable=False) fee = DB.Column(Integer, nullable=False) + confirmed_block_height = DB.Column(Integer, nullable=True) + + # Relationship to Output (outputs created by this transaction) + outputs = DB.relationship( + "Output", + back_populates="transaction", + # Explicitly reference the 'txid' column in Output + foreign_keys=[Output.txid], + ) + + # Relationship to Input (inputs spent by this transaction) + inputs = DB.relationship( + "Output", + back_populates="spent_transaction", + foreign_keys=[ + Output.spent_txid + ], # Explicitly reference the 'spent_txid' column in Output + ) + + # Relationship to Output + # inputs = DB.relationship("Output", back_populates="transaction") # at some point I will need an output relationship # prob just need it on the output diff --git a/backend/src/services/privacy_metrics/privacy_metrics.py b/backend/src/services/privacy_metrics/privacy_metrics.py index c1af3406..9c135adb 100644 --- a/backend/src/services/privacy_metrics/privacy_metrics.py +++ b/backend/src/services/privacy_metrics/privacy_metrics.py @@ -1,5 +1,5 @@ from typing import Optional -from bdkpython import bdk +from bdkpython import TransactionDetails, bdk from bitcoinlib.transactions import Transaction from src.database import DB from src.models.label import LabelName @@ -48,7 +48,9 @@ def analyze_tx_privacy( results[privacy_metric] = result elif privacy_metric == PrivacyMetricName.MINIMAL_TX_HISTORY_REVEAL: - result = cls.analyze_minimal_tx_history_reveal(txid) + result = cls.analyze_minimal_tx_history_reveal( + transaction_details=transaction_details + ) results[privacy_metric] = result elif privacy_metric == PrivacyMetricName.TIMING_ANALYSIS: @@ -92,8 +94,7 @@ def analyze_tx_privacy( results[privacy_metric] = result elif privacy_metric == PrivacyMetricName.NO_DO_NOT_SPEND_UTXOS: - result = cls.analyze_no_do_not_spend_utxos( - transaction=transaction) + result = cls.analyze_no_do_not_spend_utxos(transaction=transaction) results[privacy_metric] = result elif privacy_metric == PrivacyMetricName.NO_KYCED_UTXOS: @@ -208,7 +209,9 @@ def analyze_minimal_wealth_reveal( return True @classmethod - def analyze_minimal_tx_history_reveal(cls, txid: str) -> bool: + def analyze_minimal_tx_history_reveal( + cls, transaction_details: Optional[TransactionModel] + ) -> bool: # if only one utxo was used by this user in this tx then this automaticcaly passes since you cant use less than one utxo in a tx. # get all utxos ever for this user. # get only the ones that were created before the transaction in question. @@ -218,11 +221,65 @@ def analyze_minimal_tx_history_reveal(cls, txid: str) -> bool: # then if there are more than two of the users utxos in the original tx it will combine the top two utxos and if that is more than the utxo total contributed by the user then the metric will fail. # this will continue until it reachs the point where either the amount of utxos contributed has been tried by the top amount utxos resulting in a passing metric, or less than the amount of utxos contributed as been tried and has found that the user could have used less utxos to create the tx. Thus failing because they linked more utxos together than needed. # needed as inputs + if transaction_details is None: + return False + # how many inputs did the user include in this tx? + outputs_used_by_user_in_tx = WalletService.get_transaction_inputs_from_db( + transaction_details.txid + ) + output_used_by_user_count = len(outputs_used_by_user_in_tx) + if output_used_by_user_count == 1: + # if the user only used one output then this metric passes + # because the user used the minimum amount of outputs possible + return True + + # hmm how to get all outputs by a blockheight + # need a relationship of tx to outputs + unspent_outputs_before_this_tx = ( + WalletService.get_all_unspent_outputs_from_db_before_blockheight( + transaction_details.confirmed_block_height + ) + ) + unspent_outputs_biggest_first = sorted( + unspent_outputs_before_this_tx, key=lambda x: x.value, reverse=True + ) + + total_input_value_needed = ( + transaction_details.sent_amount - transaction_details.received_amount + ) + + # if we can get the total_input_value_needed with less outputs than the user used + # then this metric fails + current_total_value_included = 0 + current_total_utxos_included = 0 + for output in unspent_outputs_biggest_first: + current_total_value_included += output.value + current_total_utxos_included += 1 + if current_total_utxos_included >= output_used_by_user_count: + # we have not been able to use less utxos to get the same value + # therefore the minimum tx history reveal metric passes + return True + else: + if current_total_value_included >= total_input_value_needed: + # the user could have used less outputs to get the same value + # therefore this metric fails + return False + else: + # we haven't tried more outputs than the user used yet + # and we haven't reached the total_input_value_needed yet + # therefore continue to the next output + continue + + # this shouldn't be possible to reach + # but if it does return True since we couldn't make a tx with less utxos return True @classmethod def analyze_timing_analysis(cls, txid: str) -> bool: + # is same day tx is always done + # is morning, afternoon or night always done. + # aka always done in the same few hours return True @classmethod @@ -243,6 +300,8 @@ def analyze_no_small_change( if transaction_details is None: return False + LOGGER.info(f"what are confirmation time?") + total_change = ( transaction_details.sent_amount - transaction_details.received_amount ) @@ -473,7 +532,6 @@ def ensure_recently_fetched_outputs(cls) -> None: ) if last_fetched_output_datetime is None or should_refetch_outputs: - LOGGER.info( - "No last fetched output datetime found, fetching all outputs") + LOGGER.info("No last fetched output datetime found, fetching all outputs") # this will get all the outputs and add them to the database, ensuring that they exist WalletService.get_all_outputs() diff --git a/backend/src/services/wallet/wallet.py b/backend/src/services/wallet/wallet.py index 8913ff8c..abfe997d 100644 --- a/backend/src/services/wallet/wallet.py +++ b/backend/src/services/wallet/wallet.py @@ -1,7 +1,8 @@ from dataclasses import dataclass import bdkpython as bdk +from sqlalchemy.orm import aliased from bitcoinlib.transactions import Output, Transaction -from sqlalchemy import func +from sqlalchemy import func, or_ from src.models.transaction import Transaction as TransactionModel from src.models.label import Label from src.models.outputs import Output as OutputModel @@ -375,11 +376,17 @@ def add_transaction_to_db(cls, transaction_details: bdk.TransactionDetails): .first() ) if existing_transaction is None: + block_height = ( + transaction_details.confirmation_time.height + if transaction_details.confirmation_time + else None + ) new_transaction = TransactionModel( txid=transaction_details.txid, received_amount=transaction_details.received, sent_amount=transaction_details.sent, fee=transaction_details.fee, + confirmed_block_height=block_height, ) DB.session.add(new_transaction) DB.session.commit() @@ -403,6 +410,7 @@ def get_all_outputs(cls) -> List[LiveWalletOutput]: txid=transaction.txid, vout=output.output_n, address=output.address, + value=output.value, ) LastFetchedService.update_last_fetched_outputs_type() annominity_set = annominity_sets.get(output.value, 1) @@ -416,8 +424,33 @@ def get_all_outputs(cls) -> List[LiveWalletOutput]: all_outputs.append(extended_output) + # since the transactions don't come back in order we have to + # loop through them all again to mark the outputs + # that were used as inputs. + for transaction in all_transactions: + for input in transaction.inputs: + # if the input is one of my outputs then add it to the inputs + input = input.as_dict() + user_output = cls.get_output_from_db( + txid=input["prev_txid"], vout=input["output_n"] + ) + if user_output is None: + # not the users output therefore don't put it in the db + LOGGER.info("output is not the users") + else: + # add the output to the db? + # or just update the output as spend and then add what txid it was spent in? + cls.add_spend_tx_to_output( + output=user_output, txid=transaction.txid + ) + return all_outputs + @classmethod + def add_spend_tx_to_output(cls, output: OutputModel, txid: str): + output.spent_txid = txid + DB.session.commit() + # TODO add a better name since this is just adding the output to the db @classmethod def sync_local_db_with_incoming_output( @@ -425,6 +458,7 @@ def sync_local_db_with_incoming_output( txid: str, vout: int, address: str, + value: Optional[int] = None, ) -> OutputModel: """Sync the local database with the incoming output. @@ -435,12 +469,18 @@ def sync_local_db_with_incoming_output( txid=txid, vout=vout, address=address ).first() if not db_output: - db_output = cls.add_output_to_db(txid=txid, vout=vout, address=address) + db_output = cls.add_output_to_db( + txid=txid, vout=vout, address=address, value=value + ) return db_output @classmethod - def add_output_to_db(cls, vout: int, txid: str, address: str) -> OutputModel: - db_output = OutputModel(txid=txid, vout=vout, address=address, labels=[]) + def add_output_to_db( + cls, vout: int, txid: str, address: str, value: Optional[int] + ) -> OutputModel: + db_output = OutputModel( + txid=txid, vout=vout, address=address, value=value, labels=[] + ) DB.session.add(db_output) DB.session.commit() return db_output @@ -556,6 +596,51 @@ def get_transaction_outputs_from_db( return outputs + @classmethod + def get_transaction_inputs_from_db( + cls, + txid: str, + ) -> list[OutputModel]: + """Get all the outputs that were used as inputs for a txid in the db.""" + outputs_used_as_inputs = ( + OutputModel.query.join( + TransactionModel, TransactionModel.txid == OutputModel.spent_txid + ) + .filter(TransactionModel.txid == txid) + .all() + ) + + return outputs_used_as_inputs + + @classmethod + def get_all_unspent_outputs_from_db_before_blockheight( + cls, blockheight: int + ) -> List[OutputModel]: + """Get all outputs that had not been spent yet before or at a + certain block height. + + This is useful for determining which utxos were available to a wallet + at a certain blockheight when a tx was made. + """ + transaction_created = aliased(TransactionModel) + transaction_spent = aliased(TransactionModel) + unspent_outputs = ( + OutputModel.query.join( + transaction_created, transaction_created.txid == OutputModel.txid + ) + .join(transaction_spent, transaction_spent.txid == OutputModel.spent_txid) + .filter( + or_( + # use equal to as well to include all outputs used in a tx in this block + transaction_spent.confirmed_block_height <= blockheight, + transaction_spent.confirmed_block_height + == None, # Equivalent to IS NULL in SQL. # noqa: E711 + ) + ) + .all() + ) + return unspent_outputs + # TODO should this even go here or in its own service? @classmethod def add_label_to_output( From dc84eb0bc6cde9a98fb88c796ab5c23f3678b6dd Mon Sep 17 00:00:00 2001 From: Joseph Wyman Date: Sun, 10 Nov 2024 11:51:54 -0500 Subject: [PATCH 39/85] add method for no unneeded outputs --- .../privacy_metrics/privacy_metrics.py | 75 ++++++++++++++----- 1 file changed, 58 insertions(+), 17 deletions(-) diff --git a/backend/src/services/privacy_metrics/privacy_metrics.py b/backend/src/services/privacy_metrics/privacy_metrics.py index 9c135adb..5872f970 100644 --- a/backend/src/services/privacy_metrics/privacy_metrics.py +++ b/backend/src/services/privacy_metrics/privacy_metrics.py @@ -1,5 +1,4 @@ from typing import Optional -from bdkpython import TransactionDetails, bdk from bitcoinlib.transactions import Transaction from src.database import DB from src.models.label import LabelName @@ -80,7 +79,9 @@ def analyze_tx_privacy( results[privacy_metric] = result elif privacy_metric == PrivacyMetricName.NO_UNNECESSARY_INPUT: - result = cls.analyze_no_unnecessary_input(txid) + result = cls.analyze_no_unnecessary_input( + transaction_details=transaction_details + ) results[privacy_metric] = result elif privacy_metric == PrivacyMetricName.USE_MULTI_CHANGE_OUTPUTS: @@ -94,7 +95,8 @@ def analyze_tx_privacy( results[privacy_metric] = result elif privacy_metric == PrivacyMetricName.NO_DO_NOT_SPEND_UTXOS: - result = cls.analyze_no_do_not_spend_utxos(transaction=transaction) + result = cls.analyze_no_do_not_spend_utxos( + transaction=transaction) results[privacy_metric] = result elif privacy_metric == PrivacyMetricName.NO_KYCED_UTXOS: @@ -212,15 +214,10 @@ def analyze_minimal_wealth_reveal( def analyze_minimal_tx_history_reveal( cls, transaction_details: Optional[TransactionModel] ) -> bool: - # if only one utxo was used by this user in this tx then this automaticcaly passes since you cant use less than one utxo in a tx. - # get all utxos ever for this user. - # get only the ones that were created before the transaction in question. - # get the user's utxos that were used in this transaction. and calculate how much total they contributed to the transaction - # then use an algorithm that tries to get the same amount of value out of the users utxos at the time using less utxos than the ones the user used. - # this algo will order all the utxos by size, then it will start by picking the first one, if that is bigger than the total then the metric fails. - # then if there are more than two of the users utxos in the original tx it will combine the top two utxos and if that is more than the utxo total contributed by the user then the metric will fail. - # this will continue until it reachs the point where either the amount of utxos contributed has been tried by the top amount utxos resulting in a passing metric, or less than the amount of utxos contributed as been tried and has found that the user could have used less utxos to create the tx. Thus failing because they linked more utxos together than needed. - # needed as inputs + """Analyze if the user could have used less outputs from the + outputs in there wallet at the time of the tx, to cover the + amount(not including change) needed to send in the transaction. + aka revealing less of their tx history to the blockchain.""" if transaction_details is None: return False @@ -300,8 +297,6 @@ def analyze_no_small_change( if transaction_details is None: return False - LOGGER.info(f"what are confirmation time?") - total_change = ( transaction_details.sent_amount - transaction_details.received_amount ) @@ -396,8 +391,53 @@ def analyze_avoid_output_size_difference(cls, txid: str) -> bool: return True @classmethod - def analyze_no_unnecessary_input(cls, txid: str) -> bool: - # hmmm how should I do this? + def analyze_no_unnecessary_input( + cls, transaction_details: Optional[TransactionModel] + ) -> bool: + """Analyze if the user included more inputs (from the inputs used) + than needed to cover the amount. + + If a subset of the inputs that the user used in this tx could have + covered the amount needed to send (not included the change) then this metric fails. + """ + + if transaction_details is None: + return False + + total_input_value_needed = ( + transaction_details.sent_amount - transaction_details.received_amount + ) + + current_amount = 0 + current_inputs_used = 0 + + inputs_used_in_tx = WalletService.get_transaction_inputs_from_db( + transaction_details.txid + ) + inputs_used_in_tx_ordered_by_biggest_first = sorted( # type: ignore + inputs_used_in_tx, key=lambda x: x.value, reverse=True + ) + inputs_used_in_tx_count = len(inputs_used_in_tx) + + for input in inputs_used_in_tx_ordered_by_biggest_first: + current_amount += input.value + current_inputs_used += 1 + if current_amount >= total_input_value_needed: + # we have enough inputs to cover the amount needed + # and we did it using the biggest inputs first + if current_inputs_used < inputs_used_in_tx_count: + # there are still inputs left over + # therefore some were not needed + # therefore this metric fails + return False + else: + # we did not use less inputs + # therefore they were all neccessary + # therefore this metric passes + return True + + # this should not be possible to reach since the outputs + # should always be able to cover the amount sent return True @classmethod @@ -532,6 +572,7 @@ def ensure_recently_fetched_outputs(cls) -> None: ) if last_fetched_output_datetime is None or should_refetch_outputs: - LOGGER.info("No last fetched output datetime found, fetching all outputs") + LOGGER.info( + "No last fetched output datetime found, fetching all outputs") # this will get all the outputs and add them to the database, ensuring that they exist WalletService.get_all_outputs() From c2d8b685419545ea710b7c814c8dd891588f15b9 Mon Sep 17 00:00:00 2001 From: Joseph Wyman Date: Sun, 10 Nov 2024 12:25:58 -0500 Subject: [PATCH 40/85] remove a few privacy metrics I will not do, comment out a few not done yet --- backend/src/models/privacy_metric.py | 28 ++++---- .../privacy_metrics/privacy_metrics.py | 71 ++++++++----------- 2 files changed, 44 insertions(+), 55 deletions(-) diff --git a/backend/src/models/privacy_metric.py b/backend/src/models/privacy_metric.py index a95df087..a3cb7c96 100644 --- a/backend/src/models/privacy_metric.py +++ b/backend/src/models/privacy_metric.py @@ -11,14 +11,12 @@ class PrivacyMetricName(str, PyEnum): NO_ADDRESS_REUSE = "no address reuse" MINIMAL_WEALTH_REVEAL = "minimal wealth reveal" MINIMAL_TX_HISTORY_REVEAL = "minimal transaction history reveal" - TIMING_ANALYSIS = "timing analysis" # Minimize linkability of inputs and outputs through change detection. NO_CHANGE = "no change" NO_SMALL_CHANGE = "no small change" NO_ROUND_NUMBER_PAYMENTS = "rcd no round number payments" SAME_SCRIPT_TYPES = "rcd same script types" - AVOID_OUTPUT_SIZE_DIFFERENCE = "rcd avoid output size difference" NO_UNNECESSARY_INPUT = "rcd no unnecessary input heuristic" USE_MULTI_CHANGE_OUTPUTS = "rcd multi change outputs" AVOID_COMMON_CHANGE_POSITION = "avoid common change position" @@ -26,37 +24,43 @@ class PrivacyMetricName(str, PyEnum): # avoid combinding or using utxos that you do not want to be combined NO_DO_NOT_SPEND_UTXOS = "do not spend do not spends" NO_KYCED_UTXOS = "do not spend KYCED" - NO_DUST_ATTACK_UTXOS = "do not use dust attacks" - NO_POST_MIX_CHANGE = "no post mix change" - SEGREGATE_POSTMIX_AND_NONMIX = "segregate postmix and nonmix" # Add more labels as needed + # TODO metrics + # TIMING_ANALYSIS = "timing analysis" + + # NO_POST_MIX_CHANGE = "no post mix change" + # SEGREGATE_POSTMIX_AND_NONMIX = "segregate postmix and nonmix" + privacy_metrics_descriptions = { PrivacyMetricName.ANNOMINITY_SET: "A high anonymity set improves Bitcoin privacy by obscuring transaction origins and preventing address linking, making it harder for blockchain analysis to identify specific users and enhancing overall confidentiality.", PrivacyMetricName.NO_ADDRESS_REUSE: "Not reusing addresses is good for privacy because it prevents transaction history from being easily linked to a single identity, making it more difficult for observers to trace user activity and associate multiple transactions with the same individual.", PrivacyMetricName.MINIMAL_WEALTH_REVEAL: "Avoiding the use of UTXOs with large amounts is beneficial for privacy because it minimizes the risk of revealing your wealth and financial habits, making it harder for observers to assess your overall financial status or target you for theft or scams. This practice helps maintain a lower profile in transactions, reducing the likelihood of being linked to a specific identity or wealth level.", PrivacyMetricName.MINIMAL_TX_HISTORY_REVEAL: "Avoiding the use of many UTXOs helps reduce the visibility of transaction history, as consolidating or minimizing UTXOs makes it harder for observers to track and link multiple transactions to a single identity. This practice enhances privacy by obscuring spending patterns and making it difficult to establish a comprehensive financial profile based on transaction behavior.", - PrivacyMetricName.TIMING_ANALYSIS: "Avoiding timing analysis in a Bitcoin transaction enhances privacy by obscuring the relationship between senders and receivers, making it harder for observers to link transactions to specific individuals. By randomizing transaction timing or using methods like coin mixing, users can protect their financial patterns and enhance overall anonymity within the network.", PrivacyMetricName.NO_CHANGE: "Having no change in a Bitcoin transaction improves privacy by simplifying the transaction structure, making it harder for external observers to trace the flow of funds and link addresses back to their owners. This reduces the potential for address clustering, where multiple addresses are associated with the same user, thereby enhancing anonymity in the transaction history.", PrivacyMetricName.NO_SMALL_CHANGE: "Avoiding small change in Bitcoin transactions enhances privacy by minimizing the number of unspent transaction outputs (UTXOs) associated with a user's wallet, which can be traced back to them. Small UTXOs often create identifiable patterns, revealing links between inputs and outputs, and making it easier for observers to track transaction histories and associate multiple addresses with a single user, thereby compromising anonymity.", PrivacyMetricName.NO_ROUND_NUMBER_PAYMENTS: "Using non-round number payments in Bitcoin transactions helps obscure the identities of both the sender and receiver by reducing the likelihood of identifying who is involved in a transaction. Round number payments can create clear patterns, making it easier for observers to pinpoint the receiver based on the expected amounts, while irregular payment amounts complicate this analysis, making it difficult to discern the relationship between the parties and enhancing privacy for both.", PrivacyMetricName.SAME_SCRIPT_TYPES: "Maintaining the same script type between inputs and outputs in a Bitcoin transaction enhances privacy by ensuring that all elements of the transaction appear uniform and consistent, making it more challenging for observers to draw conclusions about the transaction's structure. When inputs and outputs share the same script type, it becomes harder to differentiate between them, thereby obscuring the flow of funds and reducing the potential for address clustering, which can lead to a clearer association between addresses and their owners.", - PrivacyMetricName.AVOID_OUTPUT_SIZE_DIFFERENCE: "Having similar-sized outputs in Bitcoin transactions enhances privacy by making it more difficult for observers to link specific outputs to particular inputs, as uniform output sizes create ambiguity in tracing the flow of funds. This uniformity helps obscure the relationships between addresses, reducing the likelihood of clustering and making it harder for analysts to identify patterns or associate transactions with individual users, thereby bolstering overall anonymity.", PrivacyMetricName.NO_UNNECESSARY_INPUT: "Reducing unnecessary inputs in a Bitcoin transaction is beneficial because it prevents the association of specific inputs with outputs, making it harder to trace the flow of funds. ", PrivacyMetricName.USE_MULTI_CHANGE_OUTPUTS: "Having multiple outputs in a Bitcoin transaction enhances privacy by creating a more complex transaction structure, which obscures the relationship between inputs and outputs. This added complexity makes it difficult for observers to trace the flow of funds, thereby reducing the risk of linking specific inputs to identifiable outputs.", PrivacyMetricName.AVOID_COMMON_CHANGE_POSITION: "Reusing the same change output (vout) position in Bitcoin transactions can lead to wallet clustering, where observers can group multiple addresses controlled by the same user based on transaction patterns. This compromises your privacy by making it easier to trace your spending behavior and link different transactions.", PrivacyMetricName.NO_DO_NOT_SPEND_UTXOS: "Avoiding the use of 'do not spend' UTXOs in Bitcoin transactions is essential for maintaining privacy and security, as these UTXOs are typically flagged for specific purposes and should not be combined with other funds. By avoiding these UTXOs, you can prevent unintended disclosures of sensitive information and protect your financial privacy.", PrivacyMetricName.NO_KYCED_UTXOS: "Avoiding the use of KYCed UTXOs in Bitcoin transactions is crucial for preserving privacy and anonymity, as these UTXOs are linked to your identity through Know Your Customer (KYC) processes. By excluding these UTXOs from your transactions, you can prevent the exposure of personal information and maintain a higher level of privacy and confidentiality.", - PrivacyMetricName.NO_DUST_ATTACK_UTXOS: "Avoiding the use of dust attack UTXOs in Bitcoin transactions is essential for protecting your privacy and security, as these UTXOs are often used to track and de-anonymize users through small, non-standard transactions. By excluding these UTXOs from your transactions, you can reduce the risk of being targeted by malicious actors and maintain a higher level of privacy and anonymity within the network.", - PrivacyMetricName.NO_POST_MIX_CHANGE: "Not using post mix change ensures that funds remain anonymized after mixing, preventing the re-identification of mixed funds and maintaining the privacy and anonymity provided by the mixing process. ", - PrivacyMetricName.SEGREGATE_POSTMIX_AND_NONMIX: "Segregating post-mix and non-mix funds in Bitcoin transactions is essential for maintaining privacy and anonymity, as combining these funds can undo the anonymization that the mixing provided.", } -# -# +# TODO metrics +# PrivacyMetricName.TIMING_ANALYSIS: "Avoiding timing analysis in a Bitcoin transaction enhances privacy by obscuring the relationship between senders and receivers, making it harder for observers to link transactions to specific individuals. By randomizing transaction timing or using methods like coin mixing, users can protect their financial patterns and enhance overall anonymity within the network.", + + +# PrivacyMetricName.NO_POST_MIX_CHANGE: "Not using post mix change ensures that funds remain anonymized after mixing, preventing the re-identification of mixed funds and maintaining the privacy and anonymity provided by the mixing process. ", +# PrivacyMetricName.SEGREGATE_POSTMIX_AND_NONMIX: "Segregating post-mix and non-mix funds in Bitcoin transactions is essential for maintaining privacy and anonymity, as combining these funds can undo the anonymization that the mixing provided.", +# # +# # + + class PrivacyMetric(DB.Model): __tablename__ = "privacy_metrics" # Specify the table name diff --git a/backend/src/services/privacy_metrics/privacy_metrics.py b/backend/src/services/privacy_metrics/privacy_metrics.py index 5872f970..1c3bae30 100644 --- a/backend/src/services/privacy_metrics/privacy_metrics.py +++ b/backend/src/services/privacy_metrics/privacy_metrics.py @@ -52,9 +52,10 @@ def analyze_tx_privacy( ) results[privacy_metric] = result - elif privacy_metric == PrivacyMetricName.TIMING_ANALYSIS: - result = cls.analyze_timing_analysis(txid) - results[privacy_metric] = result + # TODO + # elif privacy_metric == PrivacyMetricName.TIMING_ANALYSIS: + # result = cls.analyze_timing_analysis(txid) + # results[privacy_metric] = result elif privacy_metric == PrivacyMetricName.NO_CHANGE: result = cls.analyze_no_change(transaction_details) @@ -74,10 +75,6 @@ def analyze_tx_privacy( ) results[privacy_metric] = result - elif privacy_metric == PrivacyMetricName.AVOID_OUTPUT_SIZE_DIFFERENCE: - result = cls.analyze_avoid_output_size_difference(txid) - results[privacy_metric] = result - elif privacy_metric == PrivacyMetricName.NO_UNNECESSARY_INPUT: result = cls.analyze_no_unnecessary_input( transaction_details=transaction_details @@ -103,17 +100,14 @@ def analyze_tx_privacy( result = cls.analyze_no_kyced_inputs(transaction=transaction) results[privacy_metric] = result - elif privacy_metric == PrivacyMetricName.NO_DUST_ATTACK_UTXOS: - result = cls.analyze_no_dust_attack_utxos(txid) - results[privacy_metric] = result - - elif privacy_metric == PrivacyMetricName.NO_POST_MIX_CHANGE: - result = cls.analyze_no_post_mix_change(txid) - results[privacy_metric] = result - - elif privacy_metric == PrivacyMetricName.SEGREGATE_POSTMIX_AND_NONMIX: - result = cls.analyze_segregate_postmix_and_nonmix(txid) - results[privacy_metric] = result + # TODO + # elif privacy_metric == PrivacyMetricName.NO_POST_MIX_CHANGE: + # result = cls.analyze_no_post_mix_change(txid) + # results[privacy_metric] = result + # + # elif privacy_metric == PrivacyMetricName.SEGREGATE_POSTMIX_AND_NONMIX: + # result = cls.analyze_segregate_postmix_and_nonmix(txid) + # results[privacy_metric] = result return results @@ -272,12 +266,14 @@ def analyze_minimal_tx_history_reveal( # but if it does return True since we couldn't make a tx with less utxos return True - @classmethod - def analyze_timing_analysis(cls, txid: str) -> bool: - # is same day tx is always done - # is morning, afternoon or night always done. - # aka always done in the same few hours - return True + # # TODO this is a difficult metric, do this is a v2 + # @classmethod + # def analyze_timing_analysis(cls, txid: str) -> bool: + # # is same day tx is always done + # # is morning, afternoon or night always done. + # # aka always done in the same few hours + # return True + # @classmethod def analyze_no_change(cls, transaction_details: Optional[TransactionModel]) -> bool: @@ -384,12 +380,6 @@ def analyze_same_script_types( else: return True - @classmethod - def analyze_avoid_output_size_difference(cls, txid: str) -> bool: - # is this the same as annominity set or just looser? like the outputs shouldn't be more than 100% different or else - # it is easy to tell the change output? - return True - @classmethod def analyze_no_unnecessary_input( cls, transaction_details: Optional[TransactionModel] @@ -540,19 +530,14 @@ def analyze_no_kyced_inputs(cls, transaction: Optional[Transaction]) -> bool: return False return True - # I don't like this option any more, dust is more about fees - # a user should just mark a "tracker" output sent to them as do not spend - @classmethod - def analyze_no_dust_attack_utxos(cls, txid: str) -> bool: - return True - - @classmethod - def analyze_no_post_mix_change(cls, txid: str) -> bool: - return True - - @classmethod - def analyze_segregate_postmix_and_nonmix(cls, txid: str) -> bool: - return True + # TODO + # @classmethod + # def analyze_no_post_mix_change(cls, txid: str) -> bool: + # return True + # + # @classmethod + # def analyze_segregate_postmix_and_nonmix(cls, txid: str) -> bool: + # return True @classmethod def ensure_recently_fetched_outputs(cls) -> None: From 0ab352b903304e3a4ddb49d2f9be989bf12289f2 Mon Sep 17 00:00:00 2001 From: Joseph Wyman Date: Mon, 11 Nov 2024 09:15:16 -0500 Subject: [PATCH 41/85] add privacy method for detecting common change output position --- backend/src/models/outputs.py | 19 ++++++ .../privacy_metrics/privacy_metrics.py | 61 ++++++++++++++++- backend/src/services/wallet/wallet.py | 67 ++++++++++++++++--- 3 files changed, 135 insertions(+), 12 deletions(-) diff --git a/backend/src/models/outputs.py b/backend/src/models/outputs.py index 05d67468..8a7f0e4c 100644 --- a/backend/src/models/outputs.py +++ b/backend/src/models/outputs.py @@ -46,6 +46,25 @@ class Output(DB.Model): labels = DB.relationship( "Label", secondary=output_labels, back_populates="outputs") + @property + def is_simple_change(self) -> bool: + """Check if the output is a simple change output + + + # If the transaction it was created in the user has included inputs and also outputs + # it is change but only + # if it does not have more than one output though + # since that means it is a more complex transaction, not just simple change. + """ + is_there_change = ( + self.transaction.sent_amount > 0 and self.transaction.received_amount > 0 + ) + # if the user only has one output in the tx it has an input in + # then consider it change + is_simple_change = is_there_change and len( + self.transaction.outputs) == 1 + return is_simple_change + # Unique constraint on the combination of txid and vout __table_args__ = (DB.UniqueConstraint( "txid", "vout", name="uq_txid_vout"),) diff --git a/backend/src/services/privacy_metrics/privacy_metrics.py b/backend/src/services/privacy_metrics/privacy_metrics.py index 1c3bae30..443cbb75 100644 --- a/backend/src/services/privacy_metrics/privacy_metrics.py +++ b/backend/src/services/privacy_metrics/privacy_metrics.py @@ -88,7 +88,9 @@ def analyze_tx_privacy( results[privacy_metric] = result elif privacy_metric == PrivacyMetricName.AVOID_COMMON_CHANGE_POSITION: - result = cls.analyze_avoid_common_change_position(txid) + result = cls.analyze_avoid_common_change_position( + transaction=transaction_details + ) results[privacy_metric] = result elif privacy_metric == PrivacyMetricName.NO_DO_NOT_SPEND_UTXOS: @@ -471,8 +473,61 @@ def analyze_use_multi_change_outputs( return True @classmethod - def analyze_avoid_common_change_position(cls, txid: str) -> bool: - return True + def analyze_avoid_common_change_position( + cls, transaction: Optional[TransactionModel] + ) -> bool: + """Check that the user is not sending funds to a change output that is + an exteremly common position for them. + + + If this transaction that the user is sending funds in has + a change output with the same vout (aka position) as 80% or more of their + other change outputs then this metric fails. + + If the user has less than 8 change outputs then this metric passes + since it is not statistically relevant enough to fail this metric. + """ + if transaction is None: + return False + + change_output: Optional[OutputModel] = None + for output in transaction.outputs: + if output.is_simple_change: + change_output = output + # there should only ever be + # one simple change output in a tx + # therefore if we found one break the loop + break + + if change_output is None: + # there is no change output in this transaction + # therefore this metric passes + return True + + change_output_position = change_output.vout + all_change_outputs_count: int = WalletService.get_all_change_outputs_from_db( + None, "count" + ) + + # this metric is only statistically relevant if the user has at least 8 change outputs + if all_change_outputs_count < 8: + # not statistically relevant enough to fail this metric + return True + + only_this_vout_change_outputs_count: int = ( + WalletService.get_all_change_outputs_from_db( + change_output_position, "count" + ) + ) + + percent_this_vout_is_change_position = ( + only_this_vout_change_outputs_count / all_change_outputs_count + ) + + if percent_this_vout_is_change_position >= 0.80: + return False + else: + return True @classmethod def analyze_no_do_not_spend_utxos(cls, transaction: Optional[Transaction]) -> bool: diff --git a/backend/src/services/wallet/wallet.py b/backend/src/services/wallet/wallet.py index abfe997d..f0a8be49 100644 --- a/backend/src/services/wallet/wallet.py +++ b/backend/src/services/wallet/wallet.py @@ -164,7 +164,8 @@ def connect_wallet( ) wallet_change_descriptor = ( - bdk.Descriptor(change_descriptor, bdk.Network._value2member_map_[network]) + bdk.Descriptor(change_descriptor, + bdk.Network._value2member_map_[network]) if change_descriptor else None ) @@ -186,7 +187,8 @@ def connect_wallet( database_config=db_config, ) - LOGGER.info(f"Connecting a new wallet to electrum server {wallet_details_id}") + LOGGER.info( + f"Connecting a new wallet to electrum server {wallet_details_id}") LOGGER.info(f"xpub {wallet_descriptor.as_string()}") wallet.sync(blockchain, None) @@ -235,7 +237,8 @@ def create_spendable_descriptor( twelve_word_secret = bdk.Mnemonic(bdk.WordCount.WORDS12) # xpriv - descriptor_secret_key = bdk.DescriptorSecretKey(network, twelve_word_secret, "") + descriptor_secret_key = bdk.DescriptorSecretKey( + network, twelve_word_secret, "") wallet_descriptor = None if script_type == ScriptType.P2PKH: @@ -306,7 +309,8 @@ def get_all_transactions( LOGGER.error("No electrum wallet or wallet details found.") return [] - transactions: list[bdk.TransactionDetails] = cls.wallet.list_transactions(False) + transactions: list[bdk.TransactionDetails] = cls.wallet.list_transactions( + False) all_tx_details: List[Transaction] = [] @@ -365,7 +369,8 @@ def get_transaction(cls, txid, index=1) -> Optional[Transaction]: @classmethod def get_transaction_details(cls, txid) -> Optional[TransactionModel]: """Get the transaction details from the database.""" - transaction = DB.session.query(TransactionModel).filter_by(txid=txid).first() + transaction = DB.session.query( + TransactionModel).filter_by(txid=txid).first() return transaction @classmethod @@ -402,7 +407,8 @@ def get_all_outputs(cls) -> List[LiveWalletOutput]: all_transactions = cls.get_all_transactions() all_outputs: List[LiveWalletOutput] = [] for transaction in all_transactions: - annominity_sets = cls.calculate_output_annominity_sets(transaction.outputs) + annominity_sets = cls.calculate_output_annominity_sets( + transaction.outputs) for output in transaction.outputs: script = bdk.Script(output.script.raw) if cls.wallet and cls.wallet.is_mine(script): @@ -446,6 +452,46 @@ def get_all_outputs(cls) -> List[LiveWalletOutput]: return all_outputs + @classmethod + def get_all_change_outputs_from_db( + cls, + vout: Optional[int] = None, + query_result_type: Literal["count", "all"] = "all", + ) -> List[OutputModel] | int: + """Get all the change outputs for the current wallet. + + If a vout is supplied get all change outputs with the vout + + + This will either return a count of the change outputs or all the change outputs + depending on the query_result_type. + """ + output_alias = aliased(OutputModel) + + query = ( + DB.session.query(OutputModel) + .join(TransactionModel, TransactionModel.txid == OutputModel.txid) + .outerjoin(output_alias, output_alias.txid == OutputModel.txid) + .group_by(OutputModel.txid) + .filter( + TransactionModel.sent_amount + > 0, # Sent amount is greater than 0 so we know the user is creating change + TransactionModel.received_amount + > 0, # Received amount is greater than 0, so we know the user is getting change + ) + .having(func.count(output_alias.id) == 1) + # Only one output for the transaction because we want simple change, not any "change" from a complex tx + ) + + if vout is not None: + # only get the change outputs for a specific vout + query = query.filter(OutputModel.vout == vout) + + if query_result_type == "count": + return query.count() + else: + return query.all() + @classmethod def add_spend_tx_to_output(cls, output: OutputModel, txid: str): output.spent_txid = txid @@ -564,7 +610,8 @@ def populate_outputs_and_labels( model_dump = populate_output_labels.model_dump() for unique_output_txid_vout in model_dump.keys(): txid, vout, address = unique_output_txid_vout.split("-") - cls.sync_local_db_with_incoming_output(txid, int(vout), address) + cls.sync_local_db_with_incoming_output( + txid, int(vout), address) output_labels = model_dump[unique_output_txid_vout] for label in output_labels: display_name = label["display_name"] @@ -727,7 +774,8 @@ def build_transaction( script, amount_per_recipient_output ) - built_transaction: bdk.TxBuilderResult = tx_builder.finish(self.wallet) + built_transaction: bdk.TxBuilderResult = tx_builder.finish( + self.wallet) built_transaction.transaction_details.transaction return BuildTransactionResponseType( @@ -806,7 +854,8 @@ def get_fee_estimate_for_utxos_from_request( @classmethod def is_address_reused(self, address: str) -> bool: """Check if the address has been used in the wallet more than once.""" - outputs_with_this_address = OutputModel.query.filter_by(address=address).all() + outputs_with_this_address = OutputModel.query.filter_by( + address=address).all() address_used_count = len(outputs_with_this_address) if address_used_count > 1: From 6868bfc56f86edeeb422ca704a763a5396c86d3c Mon Sep 17 00:00:00 2001 From: Joseph Wyman Date: Tue, 12 Nov 2024 09:31:46 -0500 Subject: [PATCH 42/85] add privacy metric for not revealing too much wealth --- .../privacy_metrics/privacy_metrics.py | 82 ++++++++++++++++--- 1 file changed, 71 insertions(+), 11 deletions(-) diff --git a/backend/src/services/privacy_metrics/privacy_metrics.py b/backend/src/services/privacy_metrics/privacy_metrics.py index 443cbb75..c951e8f2 100644 --- a/backend/src/services/privacy_metrics/privacy_metrics.py +++ b/backend/src/services/privacy_metrics/privacy_metrics.py @@ -43,7 +43,9 @@ def analyze_tx_privacy( result = cls.analyze_no_address_reuse(txid) results[privacy_metric] = result elif privacy_metric == PrivacyMetricName.MINIMAL_WEALTH_REVEAL: - result = cls.analyze_minimal_wealth_reveal(txid) + result = cls.analyze_significantly_minimal_wealth_reveal( + transaction=transaction_details + ) results[privacy_metric] = result elif privacy_metric == PrivacyMetricName.MINIMAL_TX_HISTORY_REVEAL: @@ -193,18 +195,76 @@ def analyze_no_address_reuse( return True @classmethod - def analyze_minimal_wealth_reveal( + def analyze_significantly_minimal_wealth_reveal( cls, - txid: str, + transaction: Optional[TransactionModel], ) -> bool: - # hmm should this analyze every possibility to see if the user could have revealed a few less sats or - # should it just try to minimize the wealth reveal by a certain amount? like don't reveal more than 100% of the tx toal? - # ... - # get all utxos ever. - # get the ones that were in a transaction before this transaction in question. - # get how much change was - # then check if - return True + """Check how much unnecessary wealth the user revealed to the receiver + in this transaction. + + This is equivilent to how big the dollar bill you used to pay for + something was. If you paid for a 5 cent candy with a 100 dollar bill + you significantly revealed that you have a lot more wealth than that + 5 cents. If you paid for a $98 dollar purchase with a 100 dollar bill + though this would only reveal $2 of wealth, which isn't crazy to the + relatively large purchase that was made. + + This metric will be based off of how big the "spend" was compared to + how big (in terms of sats) the UTXO inputs were. + If the user reveals 1x the amount paid for an amount of 1 bitcoin or more + then this metric fails. + If the user reveals 5x the amount paid for an amount of 0.1 bitcoin or more + then this metric fails. + If the user reveals 10x the amount paid for an amount of 0.01 bitcoin or more + then this metric fails. + + + We are scaling it and not using a fixed ratio because + if you make a small purchase, 2x that purchase isn't much, but you probably + don't want to reveal 10x that purchase amount. If you make a large buy + though revealing 1x or 2x that amount would reveal a significant amount. + + + """ + if transaction is None: + return False + + if cls.analyze_no_change(transaction): + # if there is no change then the user didn't reveal any wealth + return True + + amount_sent = transaction.sent_amount + amount_sent_to_others = amount_sent - transaction.received_amount + + if amount_sent_to_others == 0: + # if it all went back to the user then + # this is not a typical simple spent tx therefore + # this metric passes + return True + ratio_threshold = cls.get_ratio_threshold(amount_sent_to_others) + acceptable_amount_to_reveal_threshold = amount_sent_to_others * ratio_threshold + + if transaction.received_amount >= acceptable_amount_to_reveal_threshold: + return False + else: + return True + + @classmethod + def get_ratio_threshold(cls, amount_sent_to_others) -> int: + # if amount needed to reveal is 1 or > then if wealth reveal is 1x then this metric fails + # if amount needed to reveal is .1 then if wealth reveal is 5x then this metric fails + # if amount needed to reveal is .01 or less then if wealth reveal is 10x then this metric fails + + # Iterate through the thresholds in descending order (from highest to lowest) + ratio_thresholds_by_amount = {100000000: 1, 10000000: 5, 1000000: 10} + for threshold, ratio in sorted( + ratio_thresholds_by_amount.items(), reverse=True + ): + if amount_sent_to_others >= threshold: + return ratio # Return the corresponding ratio if the threshold is met + + # If amount_sent_to_others is smaller than the smallest threshold, return the smallest ratio + return ratio_thresholds_by_amount[min(ratio_thresholds_by_amount)] @classmethod def analyze_minimal_tx_history_reveal( From 27c784588fb78d10801cdd24589fa88c746300d7 Mon Sep 17 00:00:00 2001 From: Joseph Wyman Date: Sat, 16 Nov 2024 06:20:12 -0500 Subject: [PATCH 43/85] fix backend tests due to privacy changes requiring additional mocks --- backend/src/controllers/wallet.py | 67 +++++++++++++++---- .../service_tests/test_wallet_service.py | 22 +++++- 2 files changed, 73 insertions(+), 16 deletions(-) diff --git a/backend/src/controllers/wallet.py b/backend/src/controllers/wallet.py index 1ed1e0bc..cc24298d 100644 --- a/backend/src/controllers/wallet.py +++ b/backend/src/controllers/wallet.py @@ -225,27 +225,68 @@ def create_spendable_wallet(): create_and_broadcast_transaction_for_bdk_wallet( wallet, blockchain, 50000, 10, p2pkh_raw_output_script ) + mine_a_block_to_miner() wallet.sync(blockchain, None) - create_and_broadcast_coinjoin_for_bdk_wallet( - wallet, - blockchain, - 50000, - 10, - [p2wsh_raw_output_script, p2pkh_raw_output_script], + create_and_broadcast_transaction_for_bdk_wallet( + wallet, blockchain, 50000, 10, p2pkh_raw_output_script ) - # Fund the wallet again so that there are a bunch of utxos - # instead of just one because the spends are spend alls. mine_a_block_to_miner() wallet.sync(blockchain, None) - randomly_fund_mock_wallet( - wallet_address.address.as_string(), - float(data.minUtxoAmount), - float(data.maxUtxoAmount), - int(data.utxoCount), + create_and_broadcast_transaction_for_bdk_wallet( + wallet, blockchain, 50000, 10, p2pkh_raw_output_script ) + mine_a_block_to_miner() + wallet.sync(blockchain, None) + create_and_broadcast_transaction_for_bdk_wallet( + wallet, blockchain, 50000, 10, p2pkh_raw_output_script + ) + + mine_a_block_to_miner() + wallet.sync(blockchain, None) + create_and_broadcast_transaction_for_bdk_wallet( + wallet, blockchain, 50000, 10, p2pkh_raw_output_script + ) + + mine_a_block_to_miner() + wallet.sync(blockchain, None) + create_and_broadcast_transaction_for_bdk_wallet( + wallet, blockchain, 50000, 10, p2pkh_raw_output_script + ) + mine_a_block_to_miner() + wallet.sync(blockchain, None) + create_and_broadcast_transaction_for_bdk_wallet( + wallet, blockchain, 50000, 10, p2pkh_raw_output_script + ) + + mine_a_block_to_miner() + wallet.sync(blockchain, None) + create_and_broadcast_transaction_for_bdk_wallet( + wallet, blockchain, 50000, 10, p2pkh_raw_output_script + ) + # mine_a_block_to_miner() + # wallet.sync(blockchain, None) + # create_and_broadcast_coinjoin_for_bdk_wallet( + # wallet, + # blockchain, + # 50000, + # 10, + # [p2wsh_raw_output_script, p2pkh_raw_output_script], + # ) + + # Fund the wallet again so that there are a bunch of utxos + # instead of just one because the spends are spend alls. + # mine_a_block_to_miner() + # wallet.sync(blockchain, None) + # randomly_fund_mock_wallet( + # wallet_address.address.as_string(), + # float(data.minUtxoAmount), + # float(data.maxUtxoAmount), + # int(data.utxoCount), + # ) + # except Exception as e: LOGGER.error("error funding wallet", error=e) return SimpleErrorResponse(message="Error funding wallet").model_dump(), 400 diff --git a/backend/src/tests/service_tests/test_wallet_service.py b/backend/src/tests/service_tests/test_wallet_service.py index b5ed75f9..f81ea866 100644 --- a/backend/src/tests/service_tests/test_wallet_service.py +++ b/backend/src/tests/service_tests/test_wallet_service.py @@ -876,7 +876,7 @@ def test_create_spendable_wallet(self): database_config=memory_mock, ) - assert response == wallet_mock + assert response == (wallet_mock, block_chain_mock) wallet_sync_mock.assert_called_with(block_chain_mock, None) def test_get_all_transactions_success(self): @@ -998,6 +998,12 @@ def test_get_all_outputs(self): patch.object( LastFetchedService, "update_last_fetched_outputs_type" ) as mock_update_last_fetched_outputs_type, + patch.object( + WalletService, "add_spend_tx_to_output" + ) as mock_add_spend_tx_to_output, + patch.object( + WalletService, "get_output_from_db" + ) as mock_get_output_from_db, ): mock_wallet.is_mine = Mock() # mark first output as mine and the second as not @@ -1029,6 +1035,8 @@ def test_get_all_outputs(self): mock_calculate_output_annominity_sets.assert_called() mock_sync_local_db_with_incoming_output.assert_called() mock_update_last_fetched_outputs_type.assert_called() + mock_add_spend_tx_to_output.assert_called() + mock_get_output_from_db.assert_called() # the returned outputs should only be the first one since we mocked out that the second output would return False when checking if it is mine @@ -1054,6 +1062,12 @@ def test_get_all_outputs_if_none_are_mine(self): patch.object( LastFetchedService, "update_last_fetched_outputs_type" ) as mock_update_last_fetched_outputs_type, + patch.object( + WalletService, "add_spend_tx_to_output" + ) as mock_add_spend_tx_to_output, + patch.object( + WalletService, "get_output_from_db", return_value=None + ) as mock_get_output_from_db, ): mock_wallet.is_mine = Mock(return_value=False) # mark No outputs as mine @@ -1081,6 +1095,8 @@ def test_get_all_outputs_if_none_are_mine(self): mock_calculate_output_annominity_sets.assert_called() mock_sync_local_db_with_incoming_output.assert_not_called() mock_update_last_fetched_outputs_type.assert_not_called() + mock_get_output_from_db.assert_called() + mock_add_spend_tx_to_output.assert_not_called() assert mock_wallet.is_mine.call_count == 2 assert get_all_outputs_response == [] @@ -1303,10 +1319,10 @@ def test_sync_local_db_with_incoming_output_for_new_output(self): mock_add_output_to_db.return_value = mock_new_output_model mock_new_last_fetched_model = Mock(return_value=mock_new_last_fetched_model) response = self.wallet_service.sync_local_db_with_incoming_output( - "txid", 0, "mock_address" + "txid", 0, "mock_address", 10 ) mock_add_output_to_db.assert_called_with( - txid="txid", vout=0, address="mock_address" + txid="txid", vout=0, address="mock_address", value=10 ) assert response == mock_new_output_model From 443761b690e5858a91c435dc23a7af77c2c5c270 Mon Sep 17 00:00:00 2001 From: Joseph Wyman Date: Sat, 16 Nov 2024 06:27:31 -0500 Subject: [PATCH 44/85] update tests for creating a mock funded wallet and comment out some code that was just for testing purposes --- backend/src/controllers/wallet.py | 120 +++++++++--------- .../test_wallet_controller.py | 30 +++-- 2 files changed, 79 insertions(+), 71 deletions(-) diff --git a/backend/src/controllers/wallet.py b/backend/src/controllers/wallet.py index cc24298d..c907f8ae 100644 --- a/backend/src/controllers/wallet.py +++ b/backend/src/controllers/wallet.py @@ -173,8 +173,7 @@ def create_spendable_wallet(): Then give the wallet a few more UTXOs. """ try: - data = CreateSpendableWalletRequestDto.model_validate_json( - request.data) + data = CreateSpendableWalletRequestDto.model_validate_json(request.data) bdk_network: bdk.Network = bdk.Network.__members__[data.network] @@ -189,8 +188,7 @@ def create_spendable_wallet(): if wallet_descriptor is None: return ( - SimpleErrorResponse( - message="Error creating wallet").model_dump(), + SimpleErrorResponse(message="Error creating wallet").model_dump(), 400, ) @@ -209,63 +207,63 @@ def create_spendable_wallet(): # make sure a few blocks are mined before continuing # to ensure the wallet is funded. - mine_a_block_to_miner() - mine_a_block_to_miner() - mine_a_block_to_miner() - - # sync the wallet so the wallet knows about latest transactions - wallet.sync(blockchain, None) - - # create and broadcast a handful of transactions - create_and_broadcast_transaction_for_bdk_wallet( - wallet, blockchain, 50000, 10, p2sh_raw_output_script - ) - mine_a_block_to_miner() - wallet.sync(blockchain, None) - create_and_broadcast_transaction_for_bdk_wallet( - wallet, blockchain, 50000, 10, p2pkh_raw_output_script - ) - - mine_a_block_to_miner() - wallet.sync(blockchain, None) - create_and_broadcast_transaction_for_bdk_wallet( - wallet, blockchain, 50000, 10, p2pkh_raw_output_script - ) - - mine_a_block_to_miner() - wallet.sync(blockchain, None) - create_and_broadcast_transaction_for_bdk_wallet( - wallet, blockchain, 50000, 10, p2pkh_raw_output_script - ) - - mine_a_block_to_miner() - wallet.sync(blockchain, None) - create_and_broadcast_transaction_for_bdk_wallet( - wallet, blockchain, 50000, 10, p2pkh_raw_output_script - ) - - mine_a_block_to_miner() - wallet.sync(blockchain, None) - create_and_broadcast_transaction_for_bdk_wallet( - wallet, blockchain, 50000, 10, p2pkh_raw_output_script - ) - - mine_a_block_to_miner() - wallet.sync(blockchain, None) - create_and_broadcast_transaction_for_bdk_wallet( - wallet, blockchain, 50000, 10, p2pkh_raw_output_script - ) - mine_a_block_to_miner() - wallet.sync(blockchain, None) - create_and_broadcast_transaction_for_bdk_wallet( - wallet, blockchain, 50000, 10, p2pkh_raw_output_script - ) - - mine_a_block_to_miner() - wallet.sync(blockchain, None) - create_and_broadcast_transaction_for_bdk_wallet( - wallet, blockchain, 50000, 10, p2pkh_raw_output_script - ) + # mine_a_block_to_miner() + # mine_a_block_to_miner() + # mine_a_block_to_miner() + # + # # sync the wallet so the wallet knows about latest transactions + # wallet.sync(blockchain, None) + # + # # create and broadcast a handful of transactions + # create_and_broadcast_transaction_for_bdk_wallet( + # wallet, blockchain, 50000, 10, p2sh_raw_output_script + # ) + # mine_a_block_to_miner() + # wallet.sync(blockchain, None) + # create_and_broadcast_transaction_for_bdk_wallet( + # wallet, blockchain, 50000, 10, p2pkh_raw_output_script + # ) + # + # mine_a_block_to_miner() + # wallet.sync(blockchain, None) + # create_and_broadcast_transaction_for_bdk_wallet( + # wallet, blockchain, 50000, 10, p2pkh_raw_output_script + # ) + # + # mine_a_block_to_miner() + # wallet.sync(blockchain, None) + # create_and_broadcast_transaction_for_bdk_wallet( + # wallet, blockchain, 50000, 10, p2pkh_raw_output_script + # ) + # + # mine_a_block_to_miner() + # wallet.sync(blockchain, None) + # create_and_broadcast_transaction_for_bdk_wallet( + # wallet, blockchain, 50000, 10, p2pkh_raw_output_script + # ) + # + # mine_a_block_to_miner() + # wallet.sync(blockchain, None) + # create_and_broadcast_transaction_for_bdk_wallet( + # wallet, blockchain, 50000, 10, p2pkh_raw_output_script + # ) + # + # mine_a_block_to_miner() + # wallet.sync(blockchain, None) + # create_and_broadcast_transaction_for_bdk_wallet( + # wallet, blockchain, 50000, 10, p2pkh_raw_output_script + # ) + # mine_a_block_to_miner() + # wallet.sync(blockchain, None) + # create_and_broadcast_transaction_for_bdk_wallet( + # wallet, blockchain, 50000, 10, p2pkh_raw_output_script + # ) + # + # mine_a_block_to_miner() + # wallet.sync(blockchain, None) + # create_and_broadcast_transaction_for_bdk_wallet( + # wallet, blockchain, 50000, 10, p2pkh_raw_output_script + # ) # mine_a_block_to_miner() # wallet.sync(blockchain, None) # create_and_broadcast_coinjoin_for_bdk_wallet( diff --git a/backend/src/tests/controller_tests/test_wallet_controller.py b/backend/src/tests/controller_tests/test_wallet_controller.py index d8357bc6..8ba5405d 100644 --- a/backend/src/tests/controller_tests/test_wallet_controller.py +++ b/backend/src/tests/controller_tests/test_wallet_controller.py @@ -160,7 +160,7 @@ def test_spendable_success(self): get_address_mock.address.as_string = MagicMock(return_value="mock_address") wallet_mock.get_address = address_mock self.mock_wallet_service.create_spendable_wallet = MagicMock( - return_value=wallet_mock + return_value=(wallet_mock, MagicMock()) ) with ( @@ -174,7 +174,9 @@ def test_spendable_success(self): return_value=spendable_descriptor_mock, ) as create_spendable_descriptor_mock, patch.object( - WalletService, "create_spendable_wallet", return_value=wallet_mock + WalletService, + "create_spendable_wallet", + return_value=(wallet_mock, MagicMock()), ) as create_spendable_wallet_mock, ): wallet_response = self.test_client.post( @@ -220,7 +222,7 @@ def test_spendable__with_descriptor_param_success(self): get_address_mock.address.as_string = MagicMock(return_value="mock_address") wallet_mock.get_address = address_mock self.mock_wallet_service.create_spendable_wallet = MagicMock( - return_value=wallet_mock + return_value=(wallet_mock, MagicMock()) ) with ( @@ -234,7 +236,9 @@ def test_spendable__with_descriptor_param_success(self): return_value=spendable_descriptor_mock, ) as create_spendable_descriptor_mock, patch.object( - WalletService, "create_spendable_wallet", return_value=wallet_mock + WalletService, + "create_spendable_wallet", + return_value=(wallet_mock, MagicMock()), ) as create_spendable_wallet_mock, patch.object( bdk, "Descriptor", return_value=spendable_descriptor_mock @@ -289,7 +293,7 @@ def test_spendable_descriptor_error(self): get_address_mock.address.as_string = MagicMock(return_value="mock_address") wallet_mock.get_address = address_mock self.mock_wallet_service.create_spendable_wallet = MagicMock( - return_value=wallet_mock + return_value=(wallet_mock, MagicMock()) ) with ( @@ -303,7 +307,9 @@ def test_spendable_descriptor_error(self): return_value=None, ) as create_spendable_descriptor_mock, patch.object( - WalletService, "create_spendable_wallet", return_value=wallet_mock + WalletService, + "create_spendable_wallet", + return_value=(wallet_mock, MagicMock()), ) as create_spendable_wallet_mock, ): wallet_response = self.test_client.post( @@ -341,7 +347,7 @@ def test_spendable_randomly_fund_mock_wallet_error(self): get_address_mock.address.as_string = MagicMock(return_value="mock_address") wallet_mock.get_address = address_mock self.mock_wallet_service.create_spendable_wallet = MagicMock( - return_value=wallet_mock + return_value=(wallet_mock, MagicMock()) ) with ( @@ -355,7 +361,9 @@ def test_spendable_randomly_fund_mock_wallet_error(self): return_value=spendable_descriptor_mock, ) as create_spendable_descriptor_mock, patch.object( - WalletService, "create_spendable_wallet", return_value=wallet_mock + WalletService, + "create_spendable_wallet", + return_value=(wallet_mock, MagicMock()), ) as create_spendable_wallet_mock, ): randomly_fund_mock_wallet_mock.side_effect = Exception("mock exception") @@ -396,7 +404,7 @@ def test_spendable_request_error(self): get_address_mock.address.as_string = MagicMock(return_value="mock_address") wallet_mock.get_address = address_mock self.mock_wallet_service.create_spendable_wallet = MagicMock( - return_value=wallet_mock + return_value=(wallet_mock, MagicMock()) ) with ( @@ -410,7 +418,9 @@ def test_spendable_request_error(self): return_value=None, ) as create_spendable_descriptor_mock, patch.object( - WalletService, "create_spendable_wallet", return_value=wallet_mock + WalletService, + "create_spendable_wallet", + return_value=(wallet_mock, MagicMock()), ) as create_spendable_wallet_mock, ): wallet_response = self.test_client.post( From 590a3789f542b7e9ab5a157c20b5e5f40b2d5c16 Mon Sep 17 00:00:00 2001 From: Joseph Wyman Date: Sat, 16 Nov 2024 06:33:49 -0500 Subject: [PATCH 45/85] fix test for getting txs --- backend/src/tests/service_tests/test_wallet_service.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/backend/src/tests/service_tests/test_wallet_service.py b/backend/src/tests/service_tests/test_wallet_service.py index f81ea866..f6904d9c 100644 --- a/backend/src/tests/service_tests/test_wallet_service.py +++ b/backend/src/tests/service_tests/test_wallet_service.py @@ -886,6 +886,12 @@ def test_get_all_transactions_success(self): "src.services.wallet.wallet.electrum_request" ) as mock_electrum_request, patch.object(WalletService, "wallet") as mock_wallet, + patch.object( + WalletService, "add_transaction_to_db" + ) as mock_add_transaction_to_db, + patch.object( + LastFetchedService, "update_last_fetched_transaction_type" + ) as mock_update_last_fetched_transaction_type, ): # mock the transactions that bdk fetches from the wallet mock_wallet.list_transactions.return_value = [ @@ -917,6 +923,8 @@ def test_get_all_transactions_success(self): get_all_transactions_response = self.wallet_service.get_all_transactions() mock_wallet.list_transactions.assert_called_with(False) + mock_add_transaction_to_db.assert_called() + mock_update_last_fetched_transaction_type.assert_called() assert mock_electrum_request.call_count == 2 mock_electrum_request.assert_any_call( From fa1758b0f8b52747c95939e6ea462f164827b92f Mon Sep 17 00:00:00 2001 From: Joseph Wyman Date: Sat, 16 Nov 2024 07:01:28 -0500 Subject: [PATCH 46/85] fix frontend test and type errors --- src/app/components/privacy/transactionsTable.tsx | 2 +- src/app/pages/Home.tsx | 7 ++++--- src/app/types/wallet.ts | 4 ++-- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/src/app/components/privacy/transactionsTable.tsx b/src/app/components/privacy/transactionsTable.tsx index 306a7863..a1042273 100644 --- a/src/app/components/privacy/transactionsTable.tsx +++ b/src/app/components/privacy/transactionsTable.tsx @@ -19,7 +19,7 @@ import { TransactionPrivacyModal } from '../TransactionPrivacyModal'; const sectionColor = 'rgb(1, 67, 97)'; type TransactionsTableProps = { - transactions: [Transaction]; + transactions: Transaction[]; btcMetric: BtcMetric; }; diff --git a/src/app/pages/Home.tsx b/src/app/pages/Home.tsx index 192e1814..52a458c9 100644 --- a/src/app/pages/Home.tsx +++ b/src/app/pages/Home.tsx @@ -68,7 +68,7 @@ function Home() { const [txMode, setTxMode] = useState(TxMode.SINGLE); const isCreateBatchTx = txMode === TxMode.BATCH; - const [displayType, setDisplayType] = useState(DisplayType.EFFICIENCY); + const [displayType, setDisplayType] = useState(DisplayType.EFFICENCY); const location = useLocation(); const { numberOfXpubs, signaturesNeeded } = location.state as { @@ -131,8 +131,9 @@ function Home() { const [minFeeScale, setMinFeeScale] = useState(minScaleOptions[0]); const [feeRate, setFeeRate] = useState(parseInt(minFeeScale.value)); // use the labels to populate the backend. - const [importedOutputLabels, setImportedOutputLabels] = - useState(null); + const [importedOutputLabels, setImportedOutputLabels] = useState< + undefined | null | GetOutputLabelsPopulateResponseType + >(null); // Initially set the current future fee rate to the current medium fee rate // if it was not set by an imported wallet. diff --git a/src/app/types/wallet.ts b/src/app/types/wallet.ts index c10ebc90..cf6f3142 100644 --- a/src/app/types/wallet.ts +++ b/src/app/types/wallet.ts @@ -1,4 +1,4 @@ -import { GetOutputLabelsResponseType, OutputLabelType } from '../api/types'; +import { GetOutputLabelsPopulateResponseType, GetOutputLabelsResponseType, OutputLabelType } from '../api/types'; import { PolicyTypeOption } from '../components/formOptions'; import { FeeRateColor, ScaleOption } from '../pages/Home'; import { BtcMetric } from './btcSatHandler'; @@ -25,7 +25,7 @@ export type Wallet = { feeScale?: ScaleOption; minFeeScale?: ScaleOption; feeRate?: string | number; - labels?: GetOutputLabelsResponseType; + labels?: GetOutputLabelsPopulateResponseType; }; export type WalletConfigs = { From d21ce4cd633082d96ed4385f772741b49376f331 Mon Sep 17 00:00:00 2001 From: Joseph Wyman Date: Sat, 16 Nov 2024 10:11:09 -0500 Subject: [PATCH 47/85] add initial tests for test privacy metrics service methods --- backend/src/models/outputs.py | 1 - .../test_privacy_metrics_service.py | 119 ++++++++++++++++++ 2 files changed, 119 insertions(+), 1 deletion(-) create mode 100644 backend/src/tests/service_tests/test_privacy_metrics_service.py diff --git a/backend/src/models/outputs.py b/backend/src/models/outputs.py index 8a7f0e4c..7bc08db8 100644 --- a/backend/src/models/outputs.py +++ b/backend/src/models/outputs.py @@ -1,6 +1,5 @@ from sqlalchemy import Integer from src.database import DB -import uuid from .output_labels import output_labels diff --git a/backend/src/tests/service_tests/test_privacy_metrics_service.py b/backend/src/tests/service_tests/test_privacy_metrics_service.py new file mode 100644 index 00000000..49066263 --- /dev/null +++ b/backend/src/tests/service_tests/test_privacy_metrics_service.py @@ -0,0 +1,119 @@ +import copy +from unittest.case import TestCase +from unittest.mock import MagicMock, patch + +from src.models.privacy_metric import PrivacyMetric, PrivacyMetricName +from src.services.wallet.wallet import WalletService +from src.tests.mocks import tx_mock +from src.services.privacy_metrics.privacy_metrics import PrivacyMetricsService + +# TODO move into mocks file? +# Mocks +privacy_metric_mock_one = MagicMock() +privacy_metric_mock_one.id = 1 +privacy_metric_mock_one.name = PrivacyMetricName.ANNOMINITY_SET +privacy_metric_mock_one.display_name = "mock_name_one" +privacy_metric_mock_one.description = "mock_description_one" + +privacy_metric_mock_two = MagicMock() +privacy_metric_mock_two.id = 2 +privacy_metric_mock_two.name = PrivacyMetricName.NO_ADDRESS_REUSE +privacy_metric_mock_two.display_name = "mock_name_two" +privacy_metric_mock_two.description = "mock_description_two" + + +class TestPrivacyMetricsService(TestCase): + def test_get_all_privacy_metrics(self): + with patch( + "src.services.privacy_metrics.privacy_metrics.DB.session.query" + ) as mock_DB_session_query: + self.mock_DB_session_query = mock_DB_session_query + mock_return_metrics = [privacy_metric_mock_one, privacy_metric_mock_two] + self.mock_DB_session_query.return_value.all.return_value = ( + mock_return_metrics + ) + response = PrivacyMetricsService.get_all_privacy_metrics() + assert response == mock_return_metrics + + def test_analyze_annominity_set_without_desired_amount(self): + with patch.object( + WalletService, "get_output_from_db" + ) as get_output_from_db_mock, patch.object( + WalletService, "calculate_output_annominity_sets" + ) as mock_calculate_output_annominity_sets: + mock_calculate_output_annominity_sets.return_value = {"1": 1, "2": 2} + # first output is the users, second isn't + get_output_from_db_mock.side_effect = [MagicMock, None] + mock_transaction_with_one_anon_set = copy.deepcopy(tx_mock) + mock_transaction_with_one_anon_set.outputs[0].value = "1" + mock_transaction_with_one_anon_set.outputs[1].value = "2" + response = PrivacyMetricsService.analyze_annominity_set( + transaction=mock_transaction_with_one_anon_set, + desired_annominity_set=2, + allow_some_uneven_change=True, + ) + assert response is False + + def test_analyze_annominity_set_with_desired_amount(self): + # test tx with less than desired fails + with patch.object( + WalletService, "get_output_from_db" + ) as get_output_from_db_mock, patch.object( + WalletService, "calculate_output_annominity_sets" + ) as mock_calculate_output_annominity_sets: + # both outputs have an anon set on 2 + mock_calculate_output_annominity_sets.return_value = {"1": 2, "2": 2} + # first output is the users, second isn't + get_output_from_db_mock.side_effect = [MagicMock, None] + mock_transaction_with_one_anon_set = copy.deepcopy(tx_mock) + mock_transaction_with_one_anon_set.outputs[0].value = "1" + mock_transaction_with_one_anon_set.outputs[1].value = "2" + + response = PrivacyMetricsService.analyze_annominity_set( + transaction=mock_transaction_with_one_anon_set, + desired_annominity_set=2, + allow_some_uneven_change=True, + ) + assert response is True + + def test_analyze_annominity_set_with_allow_uneven_change(self): + with patch.object( + WalletService, "get_output_from_db" + ) as get_output_from_db_mock, patch.object( + WalletService, "calculate_output_annominity_sets" + ) as mock_calculate_output_annominity_sets: + # both outputs are the users but only one has an anon set above the desired anon set of 2 + mock_calculate_output_annominity_sets.return_value = {"1": 2, "2": 1} + # first output is the users, second is the users change which failed anon test + get_output_from_db_mock.side_effect = [MagicMock, MagicMock] + mock_transaction_with_one_anon_set = copy.deepcopy(tx_mock) + mock_transaction_with_one_anon_set.outputs[0].value = "1" + mock_transaction_with_one_anon_set.outputs[1].value = "2" + + response = PrivacyMetricsService.analyze_annominity_set( + transaction=mock_transaction_with_one_anon_set, + desired_annominity_set=2, + allow_some_uneven_change=True, + ) + assert response is True + + def test_analyze_annominity_set_with_NOT_allow_uneven_change(self): + with patch.object( + WalletService, "get_output_from_db" + ) as get_output_from_db_mock, patch.object( + WalletService, "calculate_output_annominity_sets" + ) as mock_calculate_output_annominity_sets: + # both outputs are the users but only one has an anon set above the desired anon set of 2 + mock_calculate_output_annominity_sets.return_value = {"1": 2, "2": 1} + # first output is the users, second is the users change which failed anon test + get_output_from_db_mock.side_effect = [MagicMock, MagicMock] + mock_transaction_with_one_anon_set = copy.deepcopy(tx_mock) + mock_transaction_with_one_anon_set.outputs[0].value = "1" + mock_transaction_with_one_anon_set.outputs[1].value = "2" + + response = PrivacyMetricsService.analyze_annominity_set( + transaction=mock_transaction_with_one_anon_set, + desired_annominity_set=2, + allow_some_uneven_change=False, + ) + assert response is False From 09c2795bdc7fc8e8586b7d617ad278b4dc0cb5ca Mon Sep 17 00:00:00 2001 From: Joseph Wyman Date: Sat, 16 Nov 2024 10:47:35 -0500 Subject: [PATCH 48/85] add test for reuse addressprivacy metric --- .../test_privacy_metrics_service.py | 61 ++++++++++++++++--- 1 file changed, 52 insertions(+), 9 deletions(-) diff --git a/backend/src/tests/service_tests/test_privacy_metrics_service.py b/backend/src/tests/service_tests/test_privacy_metrics_service.py index 49066263..19323609 100644 --- a/backend/src/tests/service_tests/test_privacy_metrics_service.py +++ b/backend/src/tests/service_tests/test_privacy_metrics_service.py @@ -27,11 +27,9 @@ def test_get_all_privacy_metrics(self): with patch( "src.services.privacy_metrics.privacy_metrics.DB.session.query" ) as mock_DB_session_query: - self.mock_DB_session_query = mock_DB_session_query - mock_return_metrics = [privacy_metric_mock_one, privacy_metric_mock_two] - self.mock_DB_session_query.return_value.all.return_value = ( - mock_return_metrics - ) + mock_return_metrics = [ + privacy_metric_mock_one, privacy_metric_mock_two] + mock_DB_session_query.return_value.all.return_value = mock_return_metrics response = PrivacyMetricsService.get_all_privacy_metrics() assert response == mock_return_metrics @@ -41,7 +39,8 @@ def test_analyze_annominity_set_without_desired_amount(self): ) as get_output_from_db_mock, patch.object( WalletService, "calculate_output_annominity_sets" ) as mock_calculate_output_annominity_sets: - mock_calculate_output_annominity_sets.return_value = {"1": 1, "2": 2} + mock_calculate_output_annominity_sets.return_value = { + "1": 1, "2": 2} # first output is the users, second isn't get_output_from_db_mock.side_effect = [MagicMock, None] mock_transaction_with_one_anon_set = copy.deepcopy(tx_mock) @@ -62,7 +61,8 @@ def test_analyze_annominity_set_with_desired_amount(self): WalletService, "calculate_output_annominity_sets" ) as mock_calculate_output_annominity_sets: # both outputs have an anon set on 2 - mock_calculate_output_annominity_sets.return_value = {"1": 2, "2": 2} + mock_calculate_output_annominity_sets.return_value = { + "1": 2, "2": 2} # first output is the users, second isn't get_output_from_db_mock.side_effect = [MagicMock, None] mock_transaction_with_one_anon_set = copy.deepcopy(tx_mock) @@ -83,7 +83,8 @@ def test_analyze_annominity_set_with_allow_uneven_change(self): WalletService, "calculate_output_annominity_sets" ) as mock_calculate_output_annominity_sets: # both outputs are the users but only one has an anon set above the desired anon set of 2 - mock_calculate_output_annominity_sets.return_value = {"1": 2, "2": 1} + mock_calculate_output_annominity_sets.return_value = { + "1": 2, "2": 1} # first output is the users, second is the users change which failed anon test get_output_from_db_mock.side_effect = [MagicMock, MagicMock] mock_transaction_with_one_anon_set = copy.deepcopy(tx_mock) @@ -104,7 +105,8 @@ def test_analyze_annominity_set_with_NOT_allow_uneven_change(self): WalletService, "calculate_output_annominity_sets" ) as mock_calculate_output_annominity_sets: # both outputs are the users but only one has an anon set above the desired anon set of 2 - mock_calculate_output_annominity_sets.return_value = {"1": 2, "2": 1} + mock_calculate_output_annominity_sets.return_value = { + "1": 2, "2": 1} # first output is the users, second is the users change which failed anon test get_output_from_db_mock.side_effect = [MagicMock, MagicMock] mock_transaction_with_one_anon_set = copy.deepcopy(tx_mock) @@ -117,3 +119,44 @@ def test_analyze_annominity_set_with_NOT_allow_uneven_change(self): allow_some_uneven_change=False, ) assert response is False + + def test_analyze_no_address_reuse_passes(self): + with patch( + "src.services.privacy_metrics.privacy_metrics.OutputModel" + ) as mock_outputmodel, patch.object( + WalletService, "is_address_reused" + ) as is_address_reuse_mock: + mock_output = MagicMock() + mock_outputs = [mock_output, mock_output] + mock_outputmodel.query.filter_by.return_value.all.return_value = ( + mock_outputs + ) + + # both addresses are not reused + is_address_reuse_mock.side_effect = [False, False] + + response = PrivacyMetricsService.analyze_no_address_reuse( + txid="mock_tx_id") + assert is_address_reuse_mock.call_count == 2 + assert response is True + + def test_analyze_no_address_reuse_fails(self): + with patch( + "src.services.privacy_metrics.privacy_metrics.OutputModel" + ) as mock_outputmodel, patch.object( + WalletService, "is_address_reused" + ) as is_address_reuse_mock: + mock_output = MagicMock() + mock_outputs = [mock_output, mock_output] + mock_outputmodel.query.filter_by.return_value.all.return_value = ( + mock_outputs + ) + + # first address is reused + is_address_reuse_mock.side_effect = [True, False] + + response = PrivacyMetricsService.analyze_no_address_reuse( + txid="mock_tx_id") + # fail early after first address found is reused + assert is_address_reuse_mock.call_count == 1 + assert response is False From 65df5d9c89fe068b45e5a1534427abc9050a9fd7 Mon Sep 17 00:00:00 2001 From: Joseph Wyman Date: Sat, 16 Nov 2024 11:43:51 -0500 Subject: [PATCH 49/85] add tests for analyze significan wealth reveal --- .../test_privacy_metrics_service.py | 78 +++++++++++++++++++ 1 file changed, 78 insertions(+) diff --git a/backend/src/tests/service_tests/test_privacy_metrics_service.py b/backend/src/tests/service_tests/test_privacy_metrics_service.py index 19323609..d3ff99c6 100644 --- a/backend/src/tests/service_tests/test_privacy_metrics_service.py +++ b/backend/src/tests/service_tests/test_privacy_metrics_service.py @@ -160,3 +160,81 @@ def test_analyze_no_address_reuse_fails(self): # fail early after first address found is reused assert is_address_reuse_mock.call_count == 1 assert response is False + + def test_analyze_significantly_minimal_wealth_reveal_no_change(self): + with patch.object( + PrivacyMetricsService, "analyze_no_change" + ) as mock_analyze_no_change: + mock_analyze_no_change.return_value = True + transaction_model_mock = MagicMock() + response = ( + PrivacyMetricsService.analyze_significantly_minimal_wealth_reveal( + transaction_model_mock + ) + ) + + # if no change then no wealth is revealed + assert response is True + + def test_analyze_significantly_minimal_wealth_reveal_non_typical_tx(self): + with patch.object( + PrivacyMetricsService, "analyze_no_change" + ) as mock_analyze_no_change: + mock_analyze_no_change.return_value = True + transaction_model_mock = MagicMock() + transaction_model_mock.sent_amount = 1000000000 + transaction_model_mock.received_amount = 1000000000 + response = ( + PrivacyMetricsService.analyze_significantly_minimal_wealth_reveal( + transaction_model_mock + ) + ) + + # if nothing sent to others then no wealth is revealed + assert response is True + + def test_analyze_significantly_minimal_wealth_reveal_fails(self): + with patch.object( + PrivacyMetricsService, "analyze_no_change" + ) as mock_analyze_no_change, patch.object( + PrivacyMetricsService, "get_ratio_threshold" + ) as mock_get_ratio_threshold: + mock_analyze_no_change.return_value = False + transaction_model_mock = MagicMock() + transaction_model_mock.sent_amount = 1000000000 + transaction_model_mock.received_amount = 800000000 + # amount receiver got is 200000000 + # amount sender got in change 800000000 + # therefore revealed 4x the payment + # since threshold is 1x the payment, this fails + mock_get_ratio_threshold.return_value = 1 + + response = ( + PrivacyMetricsService.analyze_significantly_minimal_wealth_reveal( + transaction_model_mock + ) + ) + + assert response is False + + def test_analyze_significantly_minimal_wealth_reveal_passes(self): + with patch.object( + PrivacyMetricsService, "analyze_no_change" + ) as mock_analyze_no_change, patch.object( + PrivacyMetricsService, "get_ratio_threshold" + ) as mock_get_ratio_threshold: + mock_analyze_no_change.return_value = False + transaction_model_mock = MagicMock() + transaction_model_mock.sent_amount = 1000000000 + transaction_model_mock.received_amount = 1000 + # amount receiver got much more than 2x the change amount + # therefore wealth is not revealed + mock_get_ratio_threshold.return_value = 2 + + response = ( + PrivacyMetricsService.analyze_significantly_minimal_wealth_reveal( + transaction_model_mock + ) + ) + + assert response is True From 067850ad9bf0a3a25a6fb59a784ef72ab6bf50f8 Mon Sep 17 00:00:00 2001 From: Joseph Wyman Date: Sat, 16 Nov 2024 11:51:32 -0500 Subject: [PATCH 50/85] add test for ratio threshold method --- .../test_privacy_metrics_service.py | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/backend/src/tests/service_tests/test_privacy_metrics_service.py b/backend/src/tests/service_tests/test_privacy_metrics_service.py index d3ff99c6..7ee163f2 100644 --- a/backend/src/tests/service_tests/test_privacy_metrics_service.py +++ b/backend/src/tests/service_tests/test_privacy_metrics_service.py @@ -238,3 +238,27 @@ def test_analyze_significantly_minimal_wealth_reveal_passes(self): ) assert response is True + + def test_ratio_threshold(self): + one_btc = 100000000 # 1btc + point_one_btc = 10000000 # 0.1btc + point_zero_one_btc = 1000000 # 0.01btc + point_zero_zero_one_btc = 100000 # 0.001btc + + one_btc_threshold = PrivacyMetricsService.get_ratio_threshold(one_btc) + + point_one_btc_threshold = PrivacyMetricsService.get_ratio_threshold( + point_one_btc + ) + + point_zero_one_btc_threshold = PrivacyMetricsService.get_ratio_threshold( + point_zero_one_btc + ) + point_zero_zero_one_btc_threshold = PrivacyMetricsService.get_ratio_threshold( + point_zero_zero_one_btc + ) + + assert one_btc_threshold == 1 + assert point_one_btc_threshold == 5 + assert point_zero_one_btc_threshold == 10 + assert point_zero_zero_one_btc_threshold == 10 From a200f2674ebf665da6e2925cc26dc5fa02dd0f7d Mon Sep 17 00:00:00 2001 From: Joseph Wyman Date: Sat, 16 Nov 2024 12:10:18 -0500 Subject: [PATCH 51/85] add tests for metric of no change --- .../test_privacy_metrics_service.py | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/backend/src/tests/service_tests/test_privacy_metrics_service.py b/backend/src/tests/service_tests/test_privacy_metrics_service.py index 7ee163f2..4765cf27 100644 --- a/backend/src/tests/service_tests/test_privacy_metrics_service.py +++ b/backend/src/tests/service_tests/test_privacy_metrics_service.py @@ -262,3 +262,30 @@ def test_ratio_threshold(self): assert point_one_btc_threshold == 5 assert point_zero_one_btc_threshold == 10 assert point_zero_zero_one_btc_threshold == 10 + + # TODO tests for analyze_minimal_tx_history_reveal + + def test_analyze_no_change_pass(self): + mock_transaction_model = MagicMock() + mock_transaction_model.sent_amount = 1000000000 + mock_transaction_model.received_amount = 0 + response = PrivacyMetricsService.analyze_no_change( + mock_transaction_model) + assert response is True + + # is a user received but didn't send then the amount + # they received is not change, therefore no change passes + mock_transaction_model = MagicMock() + mock_transaction_model.sent_amount = 0 + mock_transaction_model.received_amount = 100000000 + response = PrivacyMetricsService.analyze_no_change( + mock_transaction_model) + assert response is True + + def test_analyze_no_change_fail(self): + mock_transaction_model = MagicMock() + mock_transaction_model.sent_amount = 1000000000 + mock_transaction_model.received_amount = 10 + response = PrivacyMetricsService.analyze_no_change( + mock_transaction_model) + assert response is False From 4863972a772b72ca2ddfb1d22054178c617f9151 Mon Sep 17 00:00:00 2001 From: Joseph Wyman Date: Sat, 16 Nov 2024 12:20:32 -0500 Subject: [PATCH 52/85] analyze no small change method --- .../test_privacy_metrics_service.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/backend/src/tests/service_tests/test_privacy_metrics_service.py b/backend/src/tests/service_tests/test_privacy_metrics_service.py index 4765cf27..e559e051 100644 --- a/backend/src/tests/service_tests/test_privacy_metrics_service.py +++ b/backend/src/tests/service_tests/test_privacy_metrics_service.py @@ -289,3 +289,19 @@ def test_analyze_no_change_fail(self): response = PrivacyMetricsService.analyze_no_change( mock_transaction_model) assert response is False + + def test_analyze_no_small_change_fail(self): + mock_transaction_model = MagicMock() + mock_transaction_model.sent_amount = 1000000000 + mock_transaction_model.received_amount = 100 + response = PrivacyMetricsService.analyze_no_small_change( + mock_transaction_model) + assert response is False + + def test_analyze_no_small_change_pass(self): + mock_transaction_model = MagicMock() + mock_transaction_model.sent_amount = 1000040000 + mock_transaction_model.received_amount = 1000000000 + response = PrivacyMetricsService.analyze_no_small_change( + mock_transaction_model) + assert response is True From 2515968902b2b06c850c3fcfbb58a2af805f4acc Mon Sep 17 00:00:00 2001 From: Joseph Wyman Date: Sat, 16 Nov 2024 12:24:31 -0500 Subject: [PATCH 53/85] add test for no round number payment --- .../test_privacy_metrics_service.py | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/backend/src/tests/service_tests/test_privacy_metrics_service.py b/backend/src/tests/service_tests/test_privacy_metrics_service.py index e559e051..8a50e9a7 100644 --- a/backend/src/tests/service_tests/test_privacy_metrics_service.py +++ b/backend/src/tests/service_tests/test_privacy_metrics_service.py @@ -305,3 +305,33 @@ def test_analyze_no_small_change_pass(self): response = PrivacyMetricsService.analyze_no_small_change( mock_transaction_model) assert response is True + + def test_analyze_no_round_number_payments_fail(self): + mock_output_one = MagicMock() + mock_output_one.value = 1000000000 + + mock_output_two = MagicMock() + mock_output_two.value = 1123432342 + + mock_transaction = MagicMock() + mock_transaction.outputs = [mock_output_one, mock_output_two] + response = PrivacyMetricsService.analyze_no_round_number_payments( + mock_transaction + ) + + assert response is False + + def test_analyze_no_round_number_payments_pass(self): + mock_output_one = MagicMock() + mock_output_one.value = 11234324532 + + mock_output_two = MagicMock() + mock_output_two.value = 1123432342 + + mock_transaction = MagicMock() + mock_transaction.outputs = [mock_output_one, mock_output_two] + response = PrivacyMetricsService.analyze_no_round_number_payments( + mock_transaction + ) + + assert response is True From 0d009c0a5407c0725e8ae7c122503e2830a360a4 Mon Sep 17 00:00:00 2001 From: Joseph Wyman Date: Sat, 16 Nov 2024 12:34:51 -0500 Subject: [PATCH 54/85] add same script type method tests --- .../test_privacy_metrics_service.py | 88 +++++++++++++++++++ 1 file changed, 88 insertions(+) diff --git a/backend/src/tests/service_tests/test_privacy_metrics_service.py b/backend/src/tests/service_tests/test_privacy_metrics_service.py index 8a50e9a7..7ed58ec0 100644 --- a/backend/src/tests/service_tests/test_privacy_metrics_service.py +++ b/backend/src/tests/service_tests/test_privacy_metrics_service.py @@ -335,3 +335,91 @@ def test_analyze_no_round_number_payments_pass(self): ) assert response is True + + def test_analyze_same_script_types_pass_if_user_not_sending(self): + mock_output_one = MagicMock() + mock_output_one.value = 11234324532 + + mock_output_two = MagicMock() + mock_output_two.value = 1123432342 + + mock_transaction = MagicMock() + mock_transaction.outputs = [mock_output_one, mock_output_two] + + mock_transaction_model = MagicMock() + # user not sending + mock_transaction_model.sent_amount = 0 + mock_transaction_model.received_amount = 1000000000 + + response = PrivacyMetricsService.analyze_same_script_types( + mock_transaction_model, mock_transaction + ) + + assert response is True + + def test_analyze_same_script_types_pass_if_no_change(self): + mock_output_one = MagicMock() + mock_output_one.value = 11234324532 + + mock_output_two = MagicMock() + mock_output_two.value = 1123432342 + + mock_transaction = MagicMock() + mock_transaction.outputs = [mock_output_one, mock_output_two] + + mock_transaction_model = MagicMock() + mock_transaction_model.sent_amount = 1000000 + # no receiving therefore no change + mock_transaction_model.received_amount = 0 + + response = PrivacyMetricsService.analyze_same_script_types( + mock_transaction_model, mock_transaction + ) + + assert response is True + + def test_analyze_same_script_types_failse_if_not_same_script_type(self): + mock_output_one = MagicMock() + mock_output_one.script_type = "p2pkh" + + mock_output_two = MagicMock() + mock_output_two.value = 1123432342 + mock_output_two.script_type = "p2wpkh" + + mock_transaction = MagicMock() + mock_transaction.outputs = [mock_output_one, mock_output_two] + + mock_transaction_model = MagicMock() + mock_transaction_model.sent_amount = 1000000 + mock_transaction_model.received_amount = 1000 + + response = PrivacyMetricsService.analyze_same_script_types( + mock_transaction_model, mock_transaction + ) + + assert response is False + + def test_analyze_same_script_types_passes_if_same_script_type(self): + mock_output_one = MagicMock() + mock_output_one.script_type = "p2pkh" + + mock_output_two = MagicMock() + mock_output_two.value = 1123432342 + mock_output_two.script_type = "p2pkh" + + mock_transaction = MagicMock() + mock_transaction.outputs = [mock_output_one, mock_output_two] + + mock_transaction_model = MagicMock() + mock_transaction_model.sent_amount = 1000000 + mock_transaction_model.received_amount = 1000 + + response = PrivacyMetricsService.analyze_same_script_types( + mock_transaction_model, mock_transaction + ) + + assert response is True + + +# TODO +# test_analyze_no_unnecessary_input From 8156ef5301597ffc5c321020a0aaab6e69cef379 Mon Sep 17 00:00:00 2001 From: Joseph Wyman Date: Sat, 16 Nov 2024 18:18:34 -0500 Subject: [PATCH 55/85] add additional privacy metric tests --- .../test_privacy_metrics_service.py | 262 +++++++++++++++++- 1 file changed, 260 insertions(+), 2 deletions(-) diff --git a/backend/src/tests/service_tests/test_privacy_metrics_service.py b/backend/src/tests/service_tests/test_privacy_metrics_service.py index 7ed58ec0..df54a214 100644 --- a/backend/src/tests/service_tests/test_privacy_metrics_service.py +++ b/backend/src/tests/service_tests/test_privacy_metrics_service.py @@ -2,6 +2,7 @@ from unittest.case import TestCase from unittest.mock import MagicMock, patch +from src.models.label import LabelName from src.models.privacy_metric import PrivacyMetric, PrivacyMetricName from src.services.wallet.wallet import WalletService from src.tests.mocks import tx_mock @@ -420,6 +421,263 @@ def test_analyze_same_script_types_passes_if_same_script_type(self): assert response is True + def test_analyze_no_unnecessary_input_fail(self): + mock_transaction_model = MagicMock() + mock_transaction_model.sent_amount = 1000000 + mock_transaction_model.received_amount = 2000000 + mock_transaction_model.txid = "mock_txid" + + with patch.object( + WalletService, "get_transaction_inputs_from_db" + ) as mock_get_transaction_inputs_from_db: + mock_input_one = MagicMock() + mock_input_one.value = 1000000 + mock_input_two = MagicMock() + + # unneccessary input + mock_input_two.value = 2000000 + mock_get_transaction_inputs_from_db.return_value = [ + mock_input_one, + mock_input_two, + ] + + response = PrivacyMetricsService.analyze_no_unnecessary_input( + mock_transaction_model + ) + + assert response is False + + def test_analyze_no_unnecessary_input_pass(self): + mock_transaction_model = MagicMock() + mock_transaction_model.sent_amount = 3001000 + mock_transaction_model.received_amount = 1000 + mock_transaction_model.txid = "mock_txid" + + with patch.object( + WalletService, "get_transaction_inputs_from_db" + ) as mock_get_transaction_inputs_from_db: + mock_input_one = MagicMock() + mock_input_one.value = 1000000 + mock_input_two = MagicMock() + + mock_input_two.value = 2001000 + mock_get_transaction_inputs_from_db.return_value = [ + mock_input_one, + mock_input_two, + ] + + response = PrivacyMetricsService.analyze_no_unnecessary_input( + mock_transaction_model + ) + + assert response is True + + def test_analyze_no_unnecessary_input_if_not_sending(self): + mock_transaction_model = MagicMock() + mock_transaction_model.sent_amount = 0 + mock_transaction_model.received_amount = 100000 + mock_transaction_model.txid = "mock_txid" + + with patch.object( + WalletService, "get_transaction_inputs_from_db" + ) as mock_get_transaction_inputs_from_db: + mock_get_transaction_inputs_from_db.return_value = [] + + response = PrivacyMetricsService.analyze_no_unnecessary_input( + mock_transaction_model + ) -# TODO -# test_analyze_no_unnecessary_input + # if no inputs contributed, then no input is unnecessary + assert response is True + + def test_analyze_use_multi_change_outputs_no_change(self): + mock_transaction_model = MagicMock() + mock_transaction_model.sent_amount = 30000 + # no change + mock_transaction_model.received_amount = 0 + mock_transaction_model.txid = "mock_txid" + + with patch.object( + WalletService, "get_transaction_outputs_from_db" + ) as mock_get_transaction_outputs_from_db: + mock_get_transaction_outputs_from_db.return_value = [] + + response = PrivacyMetricsService.analyze_use_multi_change_outputs( + mock_transaction_model + ) + + # if no change then no need to obscure the change + assert response is True + + def test_analyze_use_multi_change_outputs_pass(self): + mock_transaction_model = MagicMock() + mock_transaction_model.sent_amount = 30000 + mock_transaction_model.received_amount = 10000 + mock_transaction_model.txid = "mock_txid" + + with patch.object( + WalletService, "get_transaction_outputs_from_db" + ) as mock_get_transaction_outputs_from_db: + mock_get_transaction_outputs_from_db.return_value = [ + MagicMock(), + MagicMock(), + ] + + response = PrivacyMetricsService.analyze_use_multi_change_outputs( + mock_transaction_model + ) + + assert response is True + + def test_analyze_use_multi_change_outputs_fail(self): + mock_transaction_model = MagicMock() + mock_transaction_model.sent_amount = 30000 + mock_transaction_model.received_amount = 10000 + mock_transaction_model.txid = "mock_txid" + + with patch.object( + WalletService, "get_transaction_outputs_from_db" + ) as mock_get_transaction_outputs_from_db: + mock_get_transaction_outputs_from_db.return_value = [ + MagicMock(), + ] + + response = PrivacyMetricsService.analyze_use_multi_change_outputs( + mock_transaction_model + ) + + assert response is False + + # TODO analyze_avoid_common_change_position + + def test_analyze_no_do_not_spend_utxos_fail(self): + mock_transaction = MagicMock() + mock_transaction_input_one = MagicMock() + mock_transaction_input_one.as_dict.return_value = { + "prev_txid": "mock_txid", + "output_n": 0, + } + mock_transaction.inputs = [ + mock_transaction_input_one, + ] + + with patch.object( + WalletService, "get_output_from_db" + ) as mock_get_output_from_db: + mock_db_output_one = MagicMock() + mock_label = MagicMock() + mock_label.name = LabelName.DO_NOT_SPEND + mock_db_output_one.labels = [mock_label] + mock_get_output_from_db.return_value = mock_db_output_one + + response = PrivacyMetricsService.analyze_no_do_not_spend_utxos( + mock_transaction + ) + mock_get_output_from_db.assert_called_once_with("mock_txid", 0) + + assert response is False + + def test_analyze_no_do_not_spend_utxos_pass(self): + mock_transaction = MagicMock() + mock_transaction_input_one = MagicMock() + mock_transaction_input_one.as_dict.return_value = { + "prev_txid": "mock_txid", + "output_n": 0, + } + mock_transaction.inputs = [ + mock_transaction_input_one, + ] + + with patch.object( + WalletService, "get_output_from_db" + ) as mock_get_output_from_db: + mock_db_output_one = MagicMock() + mock_label = MagicMock() + # kyced label is not a do not spend label + mock_label.name = LabelName.KYCED + mock_db_output_one.labels = [mock_label] + mock_get_output_from_db.return_value = mock_db_output_one + + response = PrivacyMetricsService.analyze_no_do_not_spend_utxos( + mock_transaction + ) + mock_get_output_from_db.assert_called_once_with("mock_txid", 0) + + assert response is True + + def test_analyze_no_do_not_spend_utxos_when_not_user_output(self): + mock_transaction = MagicMock() + mock_transaction_input_one = MagicMock() + mock_transaction_input_one.as_dict.return_value = { + "prev_txid": "mock_txid", + "output_n": 0, + } + mock_transaction.inputs = [ + mock_transaction_input_one, + ] + + with patch.object( + WalletService, "get_output_from_db" + ) as mock_get_output_from_db: + # not users output + mock_get_output_from_db.return_value = None + + response = PrivacyMetricsService.analyze_no_do_not_spend_utxos( + mock_transaction + ) + mock_get_output_from_db.assert_called_once_with("mock_txid", 0) + + assert response is True + + def test_analyze_no_kyced_inputs_fail(self): + mock_transaction = MagicMock() + mock_transaction_input_one = MagicMock() + mock_transaction_input_one.as_dict.return_value = { + "prev_txid": "mock_txid", + "output_n": 0, + } + mock_transaction.inputs = [ + mock_transaction_input_one, + ] + + with patch.object( + WalletService, "get_output_from_db" + ) as mock_get_output_from_db: + mock_db_output_one = MagicMock() + mock_label = MagicMock() + mock_label.name = LabelName.KYCED + mock_db_output_one.labels = [mock_label] + mock_get_output_from_db.return_value = mock_db_output_one + + response = PrivacyMetricsService.analyze_no_kyced_inputs( + mock_transaction) + mock_get_output_from_db.assert_called_once_with("mock_txid", 0) + + assert response is False + + def test_analyze_no_kyced_inputs_pass(self): + mock_transaction = MagicMock() + mock_transaction_input_one = MagicMock() + mock_transaction_input_one.as_dict.return_value = { + "prev_txid": "mock_txid", + "output_n": 0, + } + mock_transaction.inputs = [ + mock_transaction_input_one, + ] + + with patch.object( + WalletService, "get_output_from_db" + ) as mock_get_output_from_db: + mock_db_output_one = MagicMock() + mock_label = MagicMock() + # not the kyc label + mock_label.name = LabelName.DO_NOT_SPEND + mock_db_output_one.labels = [mock_label] + mock_get_output_from_db.return_value = mock_db_output_one + + response = PrivacyMetricsService.analyze_no_kyced_inputs( + mock_transaction) + mock_get_output_from_db.assert_called_once_with("mock_txid", 0) + + assert response is True From bca1ad03845a27d7054cf7d5b99d00a663c53494 Mon Sep 17 00:00:00 2001 From: Joseph Wyman Date: Sun, 17 Nov 2024 10:01:35 -0500 Subject: [PATCH 56/85] add remaining initial privacy metric unit tests --- .../privacy_metrics/privacy_metrics.py | 1 - .../test_privacy_metrics_service.py | 349 +++++++++++++++++- 2 files changed, 344 insertions(+), 6 deletions(-) diff --git a/backend/src/services/privacy_metrics/privacy_metrics.py b/backend/src/services/privacy_metrics/privacy_metrics.py index c951e8f2..0fee314b 100644 --- a/backend/src/services/privacy_metrics/privacy_metrics.py +++ b/backend/src/services/privacy_metrics/privacy_metrics.py @@ -612,7 +612,6 @@ def analyze_no_do_not_spend_utxos(cls, transaction: Optional[Transaction]) -> bo # therefore it can not be labeled by the user # therefore check the next input/output continue - if LabelName.DO_NOT_SPEND in [label.name for label in users_output.labels]: return False return True diff --git a/backend/src/tests/service_tests/test_privacy_metrics_service.py b/backend/src/tests/service_tests/test_privacy_metrics_service.py index df54a214..29b3e676 100644 --- a/backend/src/tests/service_tests/test_privacy_metrics_service.py +++ b/backend/src/tests/service_tests/test_privacy_metrics_service.py @@ -1,9 +1,12 @@ import copy from unittest.case import TestCase from unittest.mock import MagicMock, patch +from datetime import datetime + from src.models.label import LabelName -from src.models.privacy_metric import PrivacyMetric, PrivacyMetricName +from src.models.privacy_metric import PrivacyMetricName +from src.services.last_fetched.last_fetched_service import LastFetchedService from src.services.wallet.wallet import WalletService from src.tests.mocks import tx_mock from src.services.privacy_metrics.privacy_metrics import PrivacyMetricsService @@ -264,8 +267,6 @@ def test_ratio_threshold(self): assert point_zero_one_btc_threshold == 10 assert point_zero_zero_one_btc_threshold == 10 - # TODO tests for analyze_minimal_tx_history_reveal - def test_analyze_no_change_pass(self): mock_transaction_model = MagicMock() mock_transaction_model.sent_amount = 1000000000 @@ -548,8 +549,6 @@ def test_analyze_use_multi_change_outputs_fail(self): assert response is False - # TODO analyze_avoid_common_change_position - def test_analyze_no_do_not_spend_utxos_fail(self): mock_transaction = MagicMock() mock_transaction_input_one = MagicMock() @@ -681,3 +680,343 @@ def test_analyze_no_kyced_inputs_pass(self): mock_get_output_from_db.assert_called_once_with("mock_txid", 0) assert response is True + + def test_ensure_recently_fetched_outputs_when_never_fetched(self): + with patch.object( + LastFetchedService, "get_last_fetched_output_datetime" + ) as mock_get_last_fetched_output_datetime, patch.object( + WalletService, "get_all_outputs" + ) as mock_get_all_outputs: + mock_get_last_fetched_output_datetime.return_value = None + response = PrivacyMetricsService.ensure_recently_fetched_outputs() + # since not recently fetched, we should fetch the outputs + mock_get_all_outputs.assert_called() + assert response == None + + def test_ensure_recently_fetched_outputs_when_not_recently_fetched(self): + with patch.object( + LastFetchedService, "get_last_fetched_output_datetime" + ) as mock_get_last_fetched_output_datetime, patch.object( + WalletService, "get_all_outputs" + ) as mock_get_all_outputs: + # 2021 is not recent + mock_get_last_fetched_output_datetime.return_value = datetime( + 2021, 1, 1) + response = PrivacyMetricsService.ensure_recently_fetched_outputs() + # since not recently fetched, we should fetch the outputs + mock_get_all_outputs.assert_called() + assert response == None + + def test_ensure_recently_fetched_outputs_when_recently_fetched(self): + with patch.object( + LastFetchedService, "get_last_fetched_output_datetime" + ) as mock_get_last_fetched_output_datetime, patch.object( + WalletService, "get_all_outputs" + ) as mock_get_all_outputs: + # now is recent + mock_get_last_fetched_output_datetime.return_value = datetime.now() + response = PrivacyMetricsService.ensure_recently_fetched_outputs() + # since recently fetched, we should NOT fetch the outputs + mock_get_all_outputs.assert_not_called() + assert response == None + + def test_analyze_avoid_common_change_position_when_no_simple_change(self): + mock_transaction_model = MagicMock() + mock_transaction_model.sent_amount = 30000 + mock_transaction_model.received_amount = 10000 + mock_transaction_model.txid = "mock_txid" + mock_output = MagicMock() + mock_output.is_simple_change = False + mock_transaction_model.outputs = [mock_output] + + response = PrivacyMetricsService.analyze_avoid_common_change_position( + mock_transaction_model + ) + # since no change position, then that means we are not using + # a common change position, therefore this metric passes + assert response is True + + def test_analyze_avoid_common_change_position_with_small_output_count(self): + mock_transaction_model = MagicMock() + mock_transaction_model.sent_amount = 30000 + mock_transaction_model.received_amount = 10000 + mock_transaction_model.txid = "mock_txid" + mock_output = MagicMock() + mock_output.is_simple_change = True + mock_output.vout = 0 + mock_transaction_model.outputs = [mock_output] + + with patch.object( + WalletService, "get_all_change_outputs_from_db" + ) as mock_get_all_change_ouotputs_from_db: + # only 7 change outputs total + mock_get_all_change_ouotputs_from_db.return_value = 7 + + response = PrivacyMetricsService.analyze_avoid_common_change_position( + mock_transaction_model + ) + mock_get_all_change_ouotputs_from_db.assert_called() + # since less than 8 outputs, then this metric passes + assert response is True + + def test_analyze_avoid_common_change_position_fails(self): + mock_transaction_model = MagicMock() + mock_transaction_model.sent_amount = 30000 + mock_transaction_model.received_amount = 10000 + mock_transaction_model.txid = "mock_txid" + mock_output = MagicMock() + mock_output.is_simple_change = True + mock_output.vout = 0 + mock_transaction_model.outputs = [mock_output] + + with patch.object( + WalletService, "get_all_change_outputs_from_db" + ) as mock_get_all_change_ouotputs_from_db: + total_change_outputs = 10 + change_outputs_as_vout_0 = 9 + mock_get_all_change_ouotputs_from_db.side_effect = [ + total_change_outputs, + change_outputs_as_vout_0, + ] + + response = PrivacyMetricsService.analyze_avoid_common_change_position( + mock_transaction_model + ) + mock_get_all_change_ouotputs_from_db.assert_called_with(0, "count") + # since 9 out of 10 change outputs are vout 0, then this metric fails + assert response is False + + def test_analyze_avoid_common_change_position_passess(self): + mock_transaction_model = MagicMock() + mock_transaction_model.sent_amount = 30000 + mock_transaction_model.received_amount = 10000 + mock_transaction_model.txid = "mock_txid" + mock_output = MagicMock() + mock_output.is_simple_change = True + mock_output.vout = 0 + mock_transaction_model.outputs = [mock_output] + + with patch.object( + WalletService, "get_all_change_outputs_from_db" + ) as mock_get_all_change_ouotputs_from_db: + total_change_outputs = 10 + change_outputs_as_vout_0 = 2 + mock_get_all_change_ouotputs_from_db.side_effect = [ + total_change_outputs, + change_outputs_as_vout_0, + ] + + response = PrivacyMetricsService.analyze_avoid_common_change_position( + mock_transaction_model + ) + mock_get_all_change_ouotputs_from_db.assert_called_with(0, "count") + # since only 2 out of 10 change outputs are vout 0, then this metric passes + assert response is True + + def test_analyze_minimal_tx_history_reveal_with_one_input(self): + with patch.object( + WalletService, "get_transaction_inputs_from_db" + ) as mock_get_transaction_inputs_from_db, patch.object( + WalletService, "get_all_unspent_outputs_from_db_before_blockheight" + ) as mock_get_all_unspent_outputs_from_db_before_blockheight: + mock_get_transaction_inputs_from_db.return_value = [MagicMock()] + + response = PrivacyMetricsService.analyze_minimal_tx_history_reveal( + MagicMock() + ) + mock_get_all_unspent_outputs_from_db_before_blockheight.assert_not_called() + # since only one input then no excess history is revealed + # therefore this metric passes + assert response is True + + def test_analyze_minimal_tx_history_reveal_fails(self): + with patch.object( + WalletService, "get_transaction_inputs_from_db" + ) as mock_get_transaction_inputs_from_db, patch.object( + WalletService, "get_all_unspent_outputs_from_db_before_blockheight" + ) as mock_get_all_unspent_outputs_from_db_before_blockheight: + input_that_covers_entire_tx = MagicMock() + input_that_covers_entire_tx.value = 2000010000 + used_input_one = MagicMock() + used_input_one.value = 1000000000 + used_input_two = MagicMock() + used_input_two.value = 1000010000 + + mock_get_transaction_inputs_from_db.return_value = [ + used_input_one, + used_input_two, + ] + + mock_get_all_unspent_outputs_from_db_before_blockheight.return_value = [ + used_input_one, + used_input_two, + input_that_covers_entire_tx, + ] + mock_transaction = MagicMock() + mock_transaction.confirmed_block_height = 100 + mock_transaction.txid = "mock_txid" + mock_transaction.sent_amount = 2000000000 + mock_transaction.received_amount = 10000 + response = PrivacyMetricsService.analyze_minimal_tx_history_reveal( + mock_transaction + ) + mock_get_transaction_inputs_from_db.assert_called_with("mock_txid") + mock_get_all_unspent_outputs_from_db_before_blockheight.assert_called_with( + 100 + ) + # since we could have used one utxo instead of two + # then this metric fails + assert response is False + + def test_analyze_minimal_tx_history_reveal_passes(self): + with patch.object( + WalletService, "get_transaction_inputs_from_db" + ) as mock_get_transaction_inputs_from_db, patch.object( + WalletService, "get_all_unspent_outputs_from_db_before_blockheight" + ) as mock_get_all_unspent_outputs_from_db_before_blockheight: + used_input_one = MagicMock() + used_input_one.value = 1000000000 + used_input_two = MagicMock() + used_input_two.value = 1000000000 + + not_used_input = MagicMock() + not_used_input.value = 1000000000 + + mock_get_transaction_inputs_from_db.return_value = [ + used_input_one, + used_input_two, + ] + + mock_get_all_unspent_outputs_from_db_before_blockheight.return_value = [ + not_used_input, + used_input_one, + used_input_two, + ] + mock_transaction = MagicMock() + mock_transaction.confirmed_block_height = 100 + mock_transaction.txid = "mock_txid" + mock_transaction.sent_amount = 2000000000 + mock_transaction.received_amount = 10000 + response = PrivacyMetricsService.analyze_minimal_tx_history_reveal( + mock_transaction + ) + mock_get_transaction_inputs_from_db.assert_called_with("mock_txid") + mock_get_all_unspent_outputs_from_db_before_blockheight.assert_called_with( + 100 + ) + # since we could not have used less than 2 utxos + # then this metric fails + assert response is True + + def test_analyze_tx_privacy_ensures_recent_outputs_call(self): + mock_txid = "mock_txid" + # empty array since we are just testing the initial calls + privacy_metrics = [] + with patch.object( + PrivacyMetricsService, "ensure_recently_fetched_outputs" + ) as mock_ensure_recently_fetched_outputs, patch.object( + WalletService, "get_transaction_details" + ) as mock_get_transaction_details, patch.object( + WalletService, "get_transaction" + ) as mock_get_transaction: + response = PrivacyMetricsService.analyze_tx_privacy( + mock_txid, privacy_metrics + ) + + mock_ensure_recently_fetched_outputs.assert_called() + mock_get_transaction.assert_called_with(mock_txid) + mock_get_transaction_details.assert_called_with(mock_txid) + + assert response == {} + + def test_analyze_tx_privacy_calls_all_metric_methods(self): + mock_txid = "mock_txid" + # empty array since we are just testing the initial calls + privacy_metrics = [ + PrivacyMetricName.ANNOMINITY_SET, + PrivacyMetricName.NO_ADDRESS_REUSE, + PrivacyMetricName.MINIMAL_WEALTH_REVEAL, + PrivacyMetricName.MINIMAL_TX_HISTORY_REVEAL, + PrivacyMetricName.NO_CHANGE, + PrivacyMetricName.NO_SMALL_CHANGE, + PrivacyMetricName.NO_ROUND_NUMBER_PAYMENTS, + PrivacyMetricName.SAME_SCRIPT_TYPES, + PrivacyMetricName.NO_UNNECESSARY_INPUT, + PrivacyMetricName.USE_MULTI_CHANGE_OUTPUTS, + PrivacyMetricName.AVOID_COMMON_CHANGE_POSITION, + PrivacyMetricName.NO_DO_NOT_SPEND_UTXOS, + PrivacyMetricName.NO_KYCED_UTXOS, + ] + + with patch.object( + PrivacyMetricsService, "ensure_recently_fetched_outputs" + ) as mock_ensure_recently_fetched_outputs, patch.object( + WalletService, "get_transaction_details" + ) as mock_get_transaction_details, patch.object( + WalletService, "get_transaction" + ) as mock_get_transaction, patch.object( + PrivacyMetricsService, "analyze_annominity_set" + ) as mock_analyze_annominity_set, patch.object( + PrivacyMetricsService, "analyze_no_address_reuse" + ) as mock_analyze_no_address_reuse, patch.object( + PrivacyMetricsService, "analyze_significantly_minimal_wealth_reveal" + ) as mock_analyze_significantly_minimal_wealth_reveal, patch.object( + PrivacyMetricsService, "analyze_minimal_tx_history_reveal" + ) as mock_analyze_minimal_tx_history_reveal, patch.object( + PrivacyMetricsService, "analyze_no_change" + ) as mock_analyze_no_change, patch.object( + PrivacyMetricsService, "analyze_no_small_change" + ) as mock_analyze_no_small_change, patch.object( + PrivacyMetricsService, "analyze_no_round_number_payments" + ) as mock_analyze_no_round_number_payments, patch.object( + PrivacyMetricsService, "analyze_same_script_types" + ) as mock_analyze_same_script_types, patch.object( + PrivacyMetricsService, "analyze_no_unnecessary_input" + ) as mock_analyze_no_unnecessary_input, patch.object( + PrivacyMetricsService, "analyze_use_multi_change_outputs" + ) as mock_analyze_use_multi_change_outputs, patch.object( + PrivacyMetricsService, "analyze_avoid_common_change_position" + ) as mock_analyze_avoid_common_change_position, patch.object( + PrivacyMetricsService, "analyze_no_do_not_spend_utxos" + ) as mock_analyze_no_do_not_spend_utxos, patch.object( + PrivacyMetricsService, "analyze_no_kyced_inputs" + ) as mock_analyze_no_kyced_inputs: + mock_analyze_no_kyced_inputs.return_value = True + mock_analyze_no_do_not_spend_utxos.return_value = True + mock_analyze_avoid_common_change_position.return_value = True + mock_analyze_use_multi_change_outputs.return_value = True + mock_analyze_no_unnecessary_input.return_value = True + mock_analyze_same_script_types.return_value = True + mock_analyze_no_round_number_payments.return_value = True + mock_analyze_no_address_reuse.return_value = True + mock_analyze_no_small_change.return_value = True + mock_analyze_no_change.return_value = True + mock_analyze_annominity_set.return_value = True + mock_analyze_significantly_minimal_wealth_reveal.return_value = True + mock_analyze_minimal_tx_history_reveal.return_value = True + mock_get_transaction_details.return_value = MagicMock() + mock_get_transaction.return_value = MagicMock() + + response = PrivacyMetricsService.analyze_tx_privacy( + mock_txid, privacy_metrics + ) + + mock_ensure_recently_fetched_outputs.assert_called() + mock_get_transaction.assert_called_with(mock_txid) + mock_get_transaction_details.assert_called_with(mock_txid) + + assert response == { + PrivacyMetricName.ANNOMINITY_SET: True, + PrivacyMetricName.NO_ADDRESS_REUSE: True, + PrivacyMetricName.MINIMAL_WEALTH_REVEAL: True, + PrivacyMetricName.MINIMAL_TX_HISTORY_REVEAL: True, + PrivacyMetricName.NO_CHANGE: True, + PrivacyMetricName.NO_SMALL_CHANGE: True, + PrivacyMetricName.NO_ROUND_NUMBER_PAYMENTS: True, + PrivacyMetricName.SAME_SCRIPT_TYPES: True, + PrivacyMetricName.NO_UNNECESSARY_INPUT: True, + PrivacyMetricName.USE_MULTI_CHANGE_OUTPUTS: True, + PrivacyMetricName.AVOID_COMMON_CHANGE_POSITION: True, + PrivacyMetricName.NO_DO_NOT_SPEND_UTXOS: True, + PrivacyMetricName.NO_KYCED_UTXOS: True, + } From 1a0a3714a1f6b11155c9f175226636a25e364ddd Mon Sep 17 00:00:00 2001 From: Joseph Wyman Date: Sun, 17 Nov 2024 11:46:58 -0500 Subject: [PATCH 57/85] add additional wallet service unit tests and find remaining db tests needed for the wallet service --- .../service_tests/test_wallet_service.py | 101 +++++++++++++++--- 1 file changed, 86 insertions(+), 15 deletions(-) diff --git a/backend/src/tests/service_tests/test_wallet_service.py b/backend/src/tests/service_tests/test_wallet_service.py index f6904d9c..5ee76d0b 100644 --- a/backend/src/tests/service_tests/test_wallet_service.py +++ b/backend/src/tests/service_tests/test_wallet_service.py @@ -117,7 +117,8 @@ def test_connect_wallet(self): descriptor_patch.assert_has_calls(expected_calls) database_config_memory_patch.assert_called() - block_chain_config_electrum_mock.assert_called_with(electrum_config_mock) + block_chain_config_electrum_mock.assert_called_with( + electrum_config_mock) electrum_config_patch.assert_called_with( wallet_details_mock.electrum_url, None, 2, 30, 100, True ) @@ -188,7 +189,8 @@ def test_connect_wallet_with_wallet_without_change_descriptor(self): descriptor_patch.assert_has_calls(expected_calls) database_config_memory_patch.assert_called() - block_chain_config_electrum_mock.assert_called_with(electrum_config_mock) + block_chain_config_electrum_mock.assert_called_with( + electrum_config_mock) electrum_config_patch.assert_called_with( wallet_details_mock.electrum_url, None, 2, 30, 100, True ) @@ -292,7 +294,8 @@ def test_connect_wallet_with_existing_wallet_but_differing_wallet_id_in_db( descriptor_patch.assert_has_calls(expected_calls) database_config_memory_patch.assert_called() - block_chain_config_electrum_mock.assert_called_with(electrum_config_mock) + block_chain_config_electrum_mock.assert_called_with( + electrum_config_mock) electrum_config_patch.assert_called_with( wallet_details_mock.electrum_url, None, 2, 30, 100, True ) @@ -341,7 +344,8 @@ def test_create_wallet(self): ): mock_wallet = MagicMock() wallet_model_patch.return_value = mock_wallet - wallet_model_patch.get_current_wallet = MagicMock(return_value=None) + wallet_model_patch.get_current_wallet = MagicMock( + return_value=None) add_mock = MagicMock() commit_mock = MagicMock() @@ -381,7 +385,8 @@ def test_create_wallet_with_no_change_descriptor(self): ): mock_wallet = MagicMock() wallet_model_patch.return_value = mock_wallet - wallet_model_patch.get_current_wallet = MagicMock(return_value=None) + wallet_model_patch.get_current_wallet = MagicMock( + return_value=None) add_mock = MagicMock() commit_mock = MagicMock() @@ -421,7 +426,8 @@ def test_create_wallet_with_existing_current_wallet_in_db(self): ): mock_wallet = MagicMock() wallet_model_patch.return_value = mock_wallet - wallet_model_patch.get_current_wallet = MagicMock(return_value=MagicMock) + wallet_model_patch.get_current_wallet = MagicMock( + return_value=MagicMock) add_mock = MagicMock() commit_mock = MagicMock() @@ -589,7 +595,8 @@ def test_build_transaction(self): output_count, ) - amount_in_each_output = (local_utxo_mock.txout.value / 2) / output_count + amount_in_each_output = ( + local_utxo_mock.txout.value / 2) / output_count tx_builder_mock.add_recipient.assert_called_with( script_mock, amount_in_each_output ) @@ -697,8 +704,10 @@ def test_get_fee_estimate_for_utxos(self): assert fee_estimate_response.status == "success" fee: int = cast(int, transaction_details_mock.fee) - expected_fee_percent = (fee / (transaction_details_mock.sent + fee)) * 100 - assert fee_estimate_response.data == FeeDetails(expected_fee_percent, fee) + expected_fee_percent = ( + fee / (transaction_details_mock.sent + fee)) * 100 + assert fee_estimate_response.data == FeeDetails( + expected_fee_percent, fee) assert fee_estimate_response.psbt == "mock_psbt" def test_get_fee_estimate_for_utxo_with_build_tx_unspendable(self): @@ -718,7 +727,8 @@ def test_get_fee_estimate_for_utxo_with_build_tx_unspendable(self): assert get_fee_estimate_response.data == None def test_get_fee_estimate_for_utxo_with_build_tx_error(self): - build_transaction_error_response = BuildTransactionResponseType("error", None) + build_transaction_error_response = BuildTransactionResponseType( + "error", None) with patch.object( WalletService, "build_transaction", @@ -863,7 +873,8 @@ def test_create_spendable_wallet(self): ) database_config_memory_patch.assert_called() - block_chain_config_electrum_mock.assert_called_with(electrum_config_mock) + block_chain_config_electrum_mock.assert_called_with( + electrum_config_mock) electrum_config_patch.assert_called_with( electrum_url_mock, None, 2, 30, 100, True ) @@ -1218,8 +1229,10 @@ def test_get_output_labels_unique(self): == mock_label_one.description ) - assert isinstance(response["txid_one-0-mockaddress1"][0], OutputLabelDto) - assert isinstance(response["txid_one-0-mockaddress1"][1], OutputLabelDto) + assert isinstance( + response["txid_one-0-mockaddress1"][0], OutputLabelDto) + assert isinstance( + response["txid_one-0-mockaddress1"][1], OutputLabelDto) def test_populate_outputs_and_labels(self): with ( @@ -1305,7 +1318,8 @@ def test_get_utxos_info(self): mock_all_utxos_three = Mock() mock_all_utxos_three.outpoint = Mock() - all_utxos_mock = [mock_all_utxos_one, mock_all_utxos_two, mock_all_utxos_three] + all_utxos_mock = [mock_all_utxos_one, + mock_all_utxos_two, mock_all_utxos_three] mock_get_all_utxos = Mock(return_value=all_utxos_mock) self.wallet_service.get_all_utxos = mock_get_all_utxos @@ -1325,7 +1339,8 @@ def test_sync_local_db_with_incoming_output_for_new_output(self): # return None, as if we did not find this OutputModel in the db mock_output_model.query.filter_by.return_value.first.return_value = None mock_add_output_to_db.return_value = mock_new_output_model - mock_new_last_fetched_model = Mock(return_value=mock_new_last_fetched_model) + mock_new_last_fetched_model = Mock( + return_value=mock_new_last_fetched_model) response = self.wallet_service.sync_local_db_with_incoming_output( "txid", 0, "mock_address", 10 ) @@ -1352,3 +1367,59 @@ def test_sync_local_db_with_incoming_output_for_existing_output(self): # since the output already exists in the db we should not call add_output_to_db mock_add_output_to_db.assert_not_called() assert response == mock_existing_output_model + + def test_is_address_reused_False(self): + with patch("src.services.wallet.wallet.OutputModel") as mock_outputmodel_query: + mock_outputmodel_query.query.filter_by.return_value.all.return_value = [ + Mock() + ] + mock_address = "mock_address" + response = self.wallet_service.is_address_reused(mock_address) + assert response is False + + def test_is_address_reused_True(self): + with patch("src.services.wallet.wallet.OutputModel") as mock_outputmodel_query: + mock_outputmodel_query.query.filter_by.return_value.all.return_value = [ + Mock(), + Mock(), + ] + mock_address = "mock_address" + response = self.wallet_service.is_address_reused(mock_address) + assert response is True + + def test_get_transaction(self): + with ( + patch("src.services.wallet.wallet.Wallet") as wallet_model_patch, + patch( + "src.services.wallet.wallet.electrum_request" + ) as mock_electrum_request, + ): + wallet_model_patch.get_current_wallet.return_value = Mock( + electrum_url="blockstream:1234" + ) + mock_electrum_request.return_value = ElectrumResponse( + status="success", data=all_transactions_mock[0] + ) + response = self.wallet_service.get_transaction("mock_txid") + mock_electrum_request.assert_called_with( + "blockstream", + 1234, + ElectrumMethod.GET_TRANSACTIONS, + GetTransactionsRequestParams("mock_txid", False), + 1, + ) + + assert response == all_transactions_mock[0] + + +# TODO do database testing for +# get_all_unspent_outputs_from_db_before_blockheight +# get_transaction_inputs_from_db +# get_transaction_outputs_from_db +# get_output_from_db +# add_output_to_db +# add_spend_tx_to_output +# get_all_change_outputs_from_db +# add_transaction_to_db +# get_transaction_details +# remove_output_and_related_label_data From ed56e8b4975062d692afabae9e34992d1af2bde9 Mon Sep 17 00:00:00 2001 From: Joseph Wyman Date: Sun, 17 Nov 2024 12:15:49 -0500 Subject: [PATCH 58/85] add tests for privacy metric controllers --- backend/src/controllers/privacy_metrics.py | 1 - .../test_privacy_metrics_controller.py | 82 +++++++++++++++++++ 2 files changed, 82 insertions(+), 1 deletion(-) create mode 100644 backend/src/tests/controller_tests/test_privacy_metrics_controller.py diff --git a/backend/src/controllers/privacy_metrics.py b/backend/src/controllers/privacy_metrics.py index 11290dd7..cca4870c 100644 --- a/backend/src/controllers/privacy_metrics.py +++ b/backend/src/controllers/privacy_metrics.py @@ -35,7 +35,6 @@ def get_privacy_metrics( Get all privacy metrics. """ try: - # TODO use a service all_metrics = privacy_service.get_all_privacy_metrics() return GetAllPrivacyMetricsResponseDto.model_validate( diff --git a/backend/src/tests/controller_tests/test_privacy_metrics_controller.py b/backend/src/tests/controller_tests/test_privacy_metrics_controller.py new file mode 100644 index 00000000..6f38d83c --- /dev/null +++ b/backend/src/tests/controller_tests/test_privacy_metrics_controller.py @@ -0,0 +1,82 @@ +from unittest import TestCase + +from unittest.mock import MagicMock, Mock +from src.app import AppCreator +import json + + +class TestPrivacyMetricsController(TestCase): + def setUp(self): + app_creator = AppCreator() + self.app = app_creator.create_app() + self.test_client = self.app.test_client() + self.mock_privacy_metrics_service = MagicMock() + + def test_get_privacy_metrics(self): + privacy_metric_mock = Mock() + privacy_metric_mock.name = "privacy_metric_1" + privacy_metric_mock.display_name = "mock_dn" + privacy_metric_mock.description = "mock_description" + + privacy_metric_mock_two = Mock() + privacy_metric_mock_two.name = "privacy_metric_2" + privacy_metric_mock_two.display_name = "mock_dn_2" + privacy_metric_mock_two.description = "mock_description_2" + + all_privacy_metrics_mock = [ + privacy_metric_mock, privacy_metric_mock_two] + + get_all_privacy_metrics_mock = MagicMock( + return_value=all_privacy_metrics_mock) + with self.app.container.privacy_metrics_service.override( + self.mock_privacy_metrics_service + ): + self.mock_privacy_metrics_service.get_all_privacy_metrics = ( + get_all_privacy_metrics_mock + ) + get_privacy_metrics_response = self.test_client.get( + "/privacy-metrics/") + + get_all_privacy_metrics_mock.assert_called_once() + + assert get_privacy_metrics_response.status == "200 OK" + assert json.loads(get_privacy_metrics_response.data) == { + "metrics": [ + { + "name": "privacy_metric_1", + "display_name": "mock_dn", + "description": "mock_description", + }, + { + "name": "privacy_metric_2", + "display_name": "mock_dn_2", + "description": "mock_description_2", + }, + ] + } + + def test_post_privacy_metrics(self): + analyze_tx_privacy_mock = MagicMock( + return_value={"annominity set": True, "no address reuse": False} + ) + + with self.app.container.privacy_metrics_service.override( + self.mock_privacy_metrics_service + ): + self.mock_privacy_metrics_service.analyze_tx_privacy = ( + analyze_tx_privacy_mock + ) + post_privacy_metrics_response = self.test_client.post( + "/privacy-metrics/", + json={ + "txid": "mock_txid", + "privacy_metrics": ["annominity set", "no address reuse"], + }, + ) + + analyze_tx_privacy_mock.assert_called_once() + + assert post_privacy_metrics_response.status == "200 OK" + assert json.loads(post_privacy_metrics_response.data) == { + "results": {"annominity set": True, "no address reuse": False} + } From 6d4c540cb4ae889a8daf4709cc2ba76eb5c67c42 Mon Sep 17 00:00:00 2001 From: Joseph Wyman Date: Sun, 17 Nov 2024 18:56:55 -0500 Subject: [PATCH 59/85] remove not kyced --- backend/src/models/label.py | 2 -- .../test_transactions_controller.py | 13 ++++++------- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/backend/src/models/label.py b/backend/src/models/label.py index 173fc432..a892ad21 100644 --- a/backend/src/models/label.py +++ b/backend/src/models/label.py @@ -9,14 +9,12 @@ class LabelName(PyEnum): DO_NOT_SPEND = "do not spend" KYCED = "kyced" - NOT_KYCED = "not kyced" # Add more labels as needed label_descriptions = { LabelName.DO_NOT_SPEND: "This output should not be spent. This can be helpful for outputs that may comprimise your privacy, like KYCED outputs, or toxic change from a coinjoin.", LabelName.KYCED: "KYC, also known as Know Your Customer, is the process of verifying the identity of customers. This output has been KYCed, which means it is attached to your name, and therefore you should be careful when spending it as it may be being monitoring.", - LabelName.NOT_KYCED: "KYC, also known as Know Your Customer, is the process of verifying the identity of customers. This output has not been KYCed, and therefore is not attached to your name, increasing your privacy.", } diff --git a/backend/src/tests/controller_tests/test_transactions_controller.py b/backend/src/tests/controller_tests/test_transactions_controller.py index 9c2a7daa..33279f92 100644 --- a/backend/src/tests/controller_tests/test_transactions_controller.py +++ b/backend/src/tests/controller_tests/test_transactions_controller.py @@ -1,18 +1,14 @@ from unittest import TestCase from unittest.mock import MagicMock, Mock -from typing import List from src.app import AppCreator -from src.models.label import Label from src.my_types.controller_types.utxos_dtos import ( OutputLabelDto, PopulateOutputLabelsRequestDto, ) -from src.my_types.transactions import LiveWalletOutput from src.services.wallet.wallet import WalletService from src.tests.mocks import all_transactions_mock, all_outputs_mock import json -from bitcoinlib.transactions import Output class TestTransactionsController(TestCase): @@ -26,7 +22,8 @@ def setUp(self): ) def test_get_transactions(self): - get_all_transactions_mock = MagicMock(return_value=all_transactions_mock) + get_all_transactions_mock = MagicMock( + return_value=all_transactions_mock) with self.app.container.wallet_service.override(self.mock_wallet_service): self.mock_wallet_service.get_all_transactions = get_all_transactions_mock get_transactions_response = self.test_client.get("/transactions/") @@ -43,7 +40,8 @@ def test_get_utxos(self): get_all_outputs_mock = MagicMock(return_value=all_outputs) with self.app.container.wallet_service.override(self.mock_wallet_service): self.mock_wallet_service.get_all_outputs = get_all_outputs_mock - get_all_outputs_response = self.test_client.get("transactions/outputs") + get_all_outputs_response = self.test_client.get( + "transactions/outputs") get_all_outputs_mock.assert_called_once() @@ -161,7 +159,8 @@ def test_get_populate_labels(self): ) mock_populate_labels = {"mock-txid-0": [output_label_mock_one]} - get_output_labels_unique_mock = MagicMock(return_value=mock_populate_labels) + get_output_labels_unique_mock = MagicMock( + return_value=mock_populate_labels) with self.app.container.wallet_service.override(self.mock_wallet_service): self.mock_wallet_service.get_output_labels_unique = ( From f49b950b50bcd84ff2e686b03bc5afbc534a386c Mon Sep 17 00:00:00 2001 From: Joseph Wyman Date: Sun, 17 Nov 2024 18:59:35 -0500 Subject: [PATCH 60/85] remove commented out code that will not be used --- backend/src/models/output_labels.py | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/backend/src/models/output_labels.py b/backend/src/models/output_labels.py index 1aca7fe2..1f9b010f 100644 --- a/backend/src/models/output_labels.py +++ b/backend/src/models/output_labels.py @@ -1,19 +1,3 @@ -# from sqlalchemy import Table, ForeignKey, Integer -# -# from src.database import DB -# -# -# class OutputLabel(DB.Model): -# __tablename__ = "output_labels" -# id = DB.Column(Integer, primary_key=True) -# output_id = DB.Column( -# "output_id", DB.String, ForeignKey("outputs.id"), primary_key=True -# ) -# label_id = DB.Column( -# "label_id", DB.String, ForeignKey("labels.id"), primary_key=True -# ) - - from sqlalchemy import Table, Column, String, ForeignKey from src.database import DB From ebdce0bb19c64c03c28e63a95e075877b23729b8 Mon Sep 17 00:00:00 2001 From: Joseph Wyman Date: Sun, 17 Nov 2024 19:09:12 -0500 Subject: [PATCH 61/85] remove commented out code that will not be used --- backend/src/models/transaction.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/backend/src/models/transaction.py b/backend/src/models/transaction.py index 2f7c0b82..d0fb2b40 100644 --- a/backend/src/models/transaction.py +++ b/backend/src/models/transaction.py @@ -31,12 +31,3 @@ class Transaction(DB.Model): Output.spent_txid ], # Explicitly reference the 'spent_txid' column in Output ) - - # Relationship to Output - # inputs = DB.relationship("Output", back_populates="transaction") - - # at some point I will need an output relationship - # prob just need it on the output - # outputs = DB.relationship( - # "Output", secondary=output_labels, back_populates="transactions" - # ) From a70de2d9e084aa8a834bc96a4047301e3e03ec84 Mon Sep 17 00:00:00 2001 From: Joseph Wyman Date: Sat, 30 Nov 2024 12:14:07 -0500 Subject: [PATCH 62/85] improve tx display stepper --- src/app/components/BitcoinAddress.tsx | 43 +++++ .../components/TransactionDetailsModal.tsx | 150 +++++++++++++++++- src/app/components/privacy/txosTable.tsx | 2 +- 3 files changed, 188 insertions(+), 7 deletions(-) create mode 100644 src/app/components/BitcoinAddress.tsx diff --git a/src/app/components/BitcoinAddress.tsx b/src/app/components/BitcoinAddress.tsx new file mode 100644 index 00000000..b3a81f2f --- /dev/null +++ b/src/app/components/BitcoinAddress.tsx @@ -0,0 +1,43 @@ +import { CopyButton, rem, Tooltip, ActionIcon } from '@mantine/core'; + +import { IconCheck, IconCopy } from '@tabler/icons-react'; +type BitcoinAddressProps = { + address: string; + splitCount?: number; +}; +export const BitcoinAddress = ({ + address, + splitCount = 4, +}: BitcoinAddressProps) => { + const prefix = address.substring(0, splitCount); + const suffix = address.substring(address?.length - splitCount); + const abrv = `${prefix}....${suffix}`; + return ( +
+ +

{abrv}

+ + + {({ copied, copy }) => ( + + + {copied ? ( + + ) : ( + + )} + + + )} + +
+ ); +}; diff --git a/src/app/components/TransactionDetailsModal.tsx b/src/app/components/TransactionDetailsModal.tsx index cdebdfa8..8d6c3a27 100644 --- a/src/app/components/TransactionDetailsModal.tsx +++ b/src/app/components/TransactionDetailsModal.tsx @@ -1,7 +1,15 @@ -import React from 'react'; -import { Modal } from '@mantine/core'; +import React, { useState } from 'react'; +import { + TextInput, + Modal, + Textarea, + Divider, + Switch, + Collapse, +} from '@mantine/core'; import { Transaction } from '../api/types'; import { BtcMetric } from '../types/btcSatHandler'; +import { BitcoinAddress } from './BitcoinAddress'; type TransactionDetailsModalProps = { opened: boolean; onClose: () => void; @@ -14,16 +22,146 @@ export const TransactionDetailsModal = ({ transactionDetails, btcMetric, }: TransactionDetailsModalProps) => { + const [showTxDiagram, setShowTxDiagram] = useState(true); return ( + Transaction ID:{' '} + {transactionDetails.txid} +

+ } > -
-

Transaction ID: {transactionDetails.txid}

+
+ +
+

Amounts

+ +
+ + + +
+
+ + +
+
+

Inputs & Outputs

+ + setShowTxDiagram(event.currentTarget.checked) + } + /> +
+ + +
+
+ {transactionDetails.inputs.map((input) => ( + + ))} +
+ + +
+
+

Inputs

+
+ +
+ Outputs +
+
+ + + +
+ {transactionDetails.outputs.map((output) => ( + + ))} +
+
+ +
+ + + +
+

Size

+
+ + + +
+
+ + + +
+

Details

+