Merge branch 'master' into feat/tx-search-conts

.github/CODEOWNERS

@@ -0,0 +1,2 @@
+haskell-src/data/* @mightybyte @sirlensalot @buckie
+haskell-src/exec/Chainweb/Coins.hs @mightybyte @sirlensalot @buckie

.github/workflows/ci.yaml

@@ -9,9 +9,9 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ghc: ['8.8.4']
+        ghc: ['8.10.7']
         cabal: ['3.4']
-        os: ['ubuntu-18.04', 'ubuntu-20.04', 'macOS-latest']
+        os: ['ubuntu-20.04', 'ubuntu-22.04', 'macOS-latest']
 
     steps:
     - name: 'GitHub actions env workaround'
@@ -23,7 +23,7 @@ jobs:
 
     # Haskell Setup
     - name: Install GHC and Cabal
-      uses: haskell/actions/setup@v1.2.3
+      uses: haskell/actions/setup@v2.3.3
       with:
          ghc-version: ${{ matrix.ghc }}
          cabal-version: ${{ matrix.cabal }}
@@ -45,15 +45,23 @@ jobs:
     - name: Update package database
       run: cabal update
     - name: Display outdated packages
-      run: cabal outdated
+      run: |
+        cd haskell-src
+        cabal outdated
     - name: Install build dependencies
-      run: cabal build --only-dependencies
+      run: |
+        cd haskell-src
+        cabal build --only-dependencies
     - name: Build
-      run: cabal build
+      run: |
+        cd haskell-src
+        cabal build
 
     # Upload artifacts
     - name: Copy build artifact
-      run: cp `cabal list-bin exe:chainweb-data` .
+      run: |
+        cd haskell-src
+        cp `cabal list-bin exe:chainweb-data` ..
     - name: Stripping binary
       run: strip chainweb-data
     - uses: actions/upload-artifact@v2
@@ -63,4 +71,6 @@ jobs:
 
     # Test
     - name: Test
-      run: cabal v2-test
+      run: |
+        cd haskell-src
+        cabal v2-test

.gitignore

@@ -1,11 +1,31 @@
-.stack-work/
-dist*
 *.swp
-*.dump-hi
 .stylish-haskell.yaml
 *.db
 result
 .nd
 cwdb-pgdata
-/TAGS
-/stack.yaml.lock
+TAGS
+dist
+dist-*
+cabal-dev
+*.o
+*.hi
+*.hie
+*.chi
+*.chs.h
+*.dyn_o
+*.dyn_hi
+.hpc
+.hsenv
+.cabal-sandbox/
+cabal.sandbox.config
+*.prof
+*.aux
+*.hp
+*.eventlog
+.stack-work/
+cabal.project.local
+cabal.project.local~
+.HTF/
+.ghc.environment.*
+

ChangeLog.md

@@ -1,6 +1,39 @@
 # Changelog
 
-## 2.0.0 (pending)
+## 2.1.1 (2023-01-23)
+
+This is a quick release after 2.1.0 for fixing an oversight in the new `/txs/accounts` endpoint.
+* Rename `chainid` -> `chain`, `name` -> `token` fields of `/txs/accounts` for consistency (#126)
+* A new `minheight` parameter for `/txs/accounts` (#127)
+* A `docker.nix` definition for building Ubuntu-based `chainweb-data` docker images using Nix (#84)
+
+## 2.1.0 (2023-01-17)
+
+_**IMPORTANT NOTICE**: Please skip this chainweb-data release and go straight to 2.1.1. Shortly after the release, we've noticed an oversight in the new `/txs/account` endpoint and decided to correct it with a quick breaking change instead of unnecessarily complicating the API. See [PR #126](https://github.com/kadena-io/chainweb-data/pull/126)._
+
+This release drops the officiall support for Ubuntu 18.04 and adds support for Ubuntu 22.04 (see #100)
+
+This is the last version that uses `beam-automigrate` for managing the database schema, from this version on, we'll switch to managing the schema using incremental migration scripts (see #101, #102, #104). When future versions of `chainweb-data` need to migrate the database from a version earlier than 2.1.0, they will ask the user to first run 2.1.0 to prepare their database for incremental migrations.
+
+- A new `/txs/account` endpoint for fetching the incoming and outgoing transfers of a Kadena or non-Kadena account. #76 (also #83, #96, #103, #110, #114, #117, #124, #125)
+- All search endpoints (`/txs/{account,events,search}`) now support an optional (at the discretion of the HTTP gateway) "bounded execution" workflow  (#109, also #118)
+- The event search endpoint `/txs/event` now accepts 2 new arguments to narrow down the search results (#74):
+   - `modulename`: Narrows down the search to events whose modules names match this value **exactly**
+   - `minheight`: The minimum block height of the search window
+- A _hidden_ new `--serve-swagger-ui` CLI argument that can be passed to `chainweb-data` to make it serve a Swagger UI for an auto-generated OpenAPI 3 spec for the `chainweb-data` HTTP API. The CLI argument is hidden because this spec is rudimentary and unofficial at the time of this release. Future releases will improve it.
+- A new `--ignore-schema-diff` CLI argument to `chainweb-data` to make it ignore any unexpected database schema changes. This can be used by `chainweb-data` operators to make schema changes to their database and keep running  `chainweb-data`, but such ad-hoc database schema changes are not officially supported since they can cause a wide variety of errors under unpredictable conditions.
+- A new `migrate` command for the `chainweb-data` CLI that can be used to run the database migrations and exit.
+- A new `/txs/txs` endpoint similar to `/txs/tx`, but it returns a list of `TxDetail` objects, which can contain more than one entry when a transaction is introduced multiple times into the blockchain on independent branches. #71 #72
+- Code search and event search query optimization (#67)
+- Add requestkey indexes on `events` and `transactions` tables (#98)
+- Refactor richlist generation (#89)
+- Load-based throttling for search endpoints (#116)
+- Optimize the recent transactions query at server start up (#119)
+- Coin circulation calculation fix #97
+- Set random_page_cost to 0 for CW-D connections #122
+
+
+## 2.0.0 (2021-08-18)
 
 This is a major backwards-incompatible update. All chainweb-data users need to
 delete their DB and rebuild from scratch.  Major changes include:

README.org

@@ -20,6 +20,7 @@
       - [[#endpoints][endpoints]]
     - [[#fill][fill]]
     - [[#backfill][backfill]]
+    - [[#backfill-transfers][backfill-transfers]]
     - [[#gaps][gaps]]
     - [[#single][single]]
 
@@ -31,7 +32,9 @@ can easily determine mining statistics and confirm transaction contents.
 
 * Requirements
 
-~chainweb-data~ requires [[https://www.postgresql.org/][Postgres]].
+~chainweb-data~ requires [[https://www.postgresql.org/][Postgres]]. If you plan to host a chainweb-data instance
+on a cloud machine (e.g. Amazon EC2), we recommend that you run the postgres
+instance on an instance attached storage unit.
 
 * Building 
 
@@ -110,6 +113,9 @@ chainweb:
     global: 1000
 #+end_example
 
+You can find an example node config in this repository in
+[node-config-for-chainweb-data.yaml](node-config-for-chainweb-data.yaml).
+
 ** How to run chainweb-data
 
 When running chainweb-data for the first time you should run ~chainweb-data
@@ -143,17 +149,41 @@ As a new block comes in, its chain number is printed as a single digit.
 ~server~ is just like ~listen~ but also runs an HTTP server that serves a
 few endpoints for doing common queries.
 
-**** endpoints
+**** Endpoints
 
 - ~/txs/recent~ gets a list of recent transactions
 - ~/txs/search?search=foo&limit=20&offset=40~ searches for transactions containing the string ~foo~
 - ~/txs/tx?requestkey=<request-key>~ gets the details of a transaction with the given request key
 - ~/txs/events?search=foo&limit=20&offset=40~ gets the details of a transaction with the given request key
 - ~/stats~ returns a few stats such as transaction count and coins in circulation
 - ~/coins~ returns just the coins in circulation
+- ~/txs/account/<account-identifier>?token=<token-name>&chainid=<chainid>&fromheight=<blockheight>&limit=<some-limit>&offset=<some-offset>~
+  returns account information given some ~account-identifier~, ~token~ and
+  ~chainid~. The optional parameter ~minheight~ forces the results to only have
+  blockheights larger than than or equal to it. If ~token~ is omitted, the token ~coin~
+  is assumed. If ~chainid~ is omitted, all chains are searched.
 
 For more detailed information, see the API definition [[https://github.com/kadena-io/chainweb-api/blob/master/lib/ChainwebData/Api.hs#L24][here]].
 
+**** Note about partial search results
+
+All of ~chainweb-data~'s search endpoints (~/txs/{events,search,account}~) support a common workflow
+for efficiently retrieving the results of a given search in non-overlapping batches.
+
+A request to any one of these endpoints that match more rows than the number asked with the ~limit~
+query parameter will respond with a ~Chainweb-Next~ response header containing a token. That token
+can be used to call the same endpoint with the same query parameters plus the token passed in via
+the ~next~ query parameter in order to retreive the next batch of results.
+
+~chainweb-data~ supports a ~Chainweb-Execution-Strategy~ request header that can be used (probably by
+~chainweb-data~ operators by setting it in the API gateway) to enable
+an upper bound on the amount of time the server will spend for searching results. Normally, the
+search endpoints will produce the given ~limit~-many results if the search matches at least that many
+entries. However, if ~Chainweb-Execution-Strategy: Bounded~ is passed in, the response can contain
+less than ~limit~ rows even though there are potentially more matches, if those matches aren't found
+quickly enough. In such a case, the returned ~Chainweb-Next~ token will act as a cursor for the search,
+so it's possible to keep searching by making successive calls with subsequent ~Chainweb-Next~ tokens.
+
 *** fill
 
 ~fill~ fills in missing blocks. This command used to be called ~gaps~ but it has
@@ -185,6 +215,17 @@ empty chains, it won't proceed.
 
 ~backfill~ will stop when it reaches height 0.
 
+*** backfill-transfers
+
+~backfill-transfers~ fills entries in the transfers table from the highest block
+height it can find for each chain up until the height that events for coinbase
+transfers began to exist.
+
+*Note:* If the transfers table is empty, you must fetch at least one row for each
+chain first via ~listen~ before doing ~backfill-transfers~! If ~backfill-transfers~ detects any
+empty chains, it won't proceed.
+
+
 *** gaps
 
 *Deprecated:* The backfill command is deprecated and will be removed in future

cabal.project

@@ -1,4 +1,4 @@
-packages: chainweb-data.cabal
+packages: haskell-src/chainweb-data.cabal
 
 package aeson
     flags: +cffi
@@ -7,21 +7,25 @@ source-repository-package
     type: git
     location: https://github.com/mightybyte/beam-automigrate.git
     tag: 112c39953c432b05ec6ae2354b0150c61ee30157
+    --sha256: sha256-0x8L1moEyayqdxHUVpYvvdvMcBCo6qMXku2zDzpMNq8=
 
 source-repository-package
     type: git
     location: https://github.com/kadena-io/pact.git
-    tag: 8681a6d6e72eccefe00100f86202d616ab5d1621
+    tag: 957b8bd7644cebc60d043b33f0d7ffc53c65d783
+    --sha256: sha256-ERyHLWEoV83mzjSlMwY8WmHsEaq3JrbxFwQjW0tFXLE=
 
 source-repository-package
     type: git
     location: https://github.com/kadena-io/chainweb-api.git
-    tag: d6ad27ed16c060e18bd86213d04d15a360f88d35
+    tag: 8a4731c2875753617ccd2b573cf726fa100c6053
+    --sha256: sha256-MY470zn+BveL7X7gFVSgGqvWjD0jsk6VKRF/gzal9Bc=
 
 source-repository-package
     type: git
     location: https://github.com/tathougies/beam.git
     tag: 596981a1ea6765b9f311d48a2ec4d8460ebc4b7e
+    --sha256: sha256-4jfhHJTUKu09ULNFFACh3v9r3S0sJkUjTyjXq6gJY2A=
     subdir:
       beam-core
       beam-migrate
@@ -31,6 +35,7 @@ source-repository-package
     type: git
     location: https://github.com/obsidiansystems/gargoyle
     tag: df0068f9572c1371bed7aa416af84d462c3574c0
+    --sha256: sha256-1NWmQf0zgi4RtQISXvk0JbsQ1OkpYJ789pkglE3fVaU=
     subdir:
       gargoyle
       gargoyle-postgresql
@@ -39,13 +44,11 @@ source-repository-package
     type: git
     location: https://github.com/kadena-io/thyme.git
     tag: 6ee9fcb026ebdb49b810802a981d166680d867c9
+    --sha256: sha256-DqGIoPGBg8py044Xa7l/Y09K+cKGFwlxONHoZRJyzCU=
 
 package vault
     documentation: false
 
 write-ghc-environment-files: never
 
-constraints:
-    sbv == 8.8
-
 allow-newer: gargoyle:base