diff --git a/.github/workflows/create-hotfix-tag.yml b/.github/workflows/create-hotfix-tag.yml new file mode 100644 index 000000000000..45699bda24dc --- /dev/null +++ b/.github/workflows/create-hotfix-tag.yml @@ -0,0 +1,100 @@ +name: Create tag on hotfix branch + +on: + workflow_dispatch: + +jobs: + create_tag: + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + with: + fetch-depth: 0 + token: ${{ secrets.AUTO_RELEASE_PAT }} + + - name: Install git-cliff + uses: baptiste0928/cargo-install@v2.1.0 + with: + crate: git-cliff + version: 1.2.0 + + - name: Check if the input is valid hotfix branch + shell: bash + run: | + if [[ ${{github.ref}} =~ ^refs/heads/hotfix-[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "::notice::${{github.ref}} is a valid branch." + else + echo "::error::${{github.ref}} is not a valid branch." + exit 1 + fi + + - name: Check if the latest commit is tag + shell: bash + run: | + if [[ -z "$(git tag --points-at HEAD)" ]]; then + echo "::notice::The latest commit is not a tag " + else + echo "::error::The latest commit on the branch is already a tag" + exit 1 + fi + + - name: Determine current and next tag + shell: bash + run: | + function get_next_tag() { + local previous_tag="${1}" + local previous_hotfix_number + local next_tag + + previous_hotfix_number="$(echo "${previous_tag}" | awk -F. '{ print $4 }')" + + if [[ -z "${previous_hotfix_number}" ]]; then + # Previous tag was not a hotfix tag + next_tag="${previous_tag}+hotfix.1" + else + # Previous tag was a hotfix tag, increment hotfix number + local hotfix_number=$((previous_hotfix_number + 1)) + next_tag="${previous_tag/%${previous_hotfix_number}/${hotfix_number}}" + fi + + echo "${next_tag}" + } + + PREVIOUS_TAG="$(git tag --merged | sort --version-sort | tail --lines 1)" + NEXT_TAG="$(get_next_tag "${PREVIOUS_TAG}")" + + echo "PREVIOUS_TAG=${PREVIOUS_TAG}" >> $GITHUB_ENV + echo "NEXT_TAG=${NEXT_TAG}" >> $GITHUB_ENV + + - name: Generate changelog + shell: bash + run: | + # Generate changelog content and store it in `release-notes.md` + git-cliff --config '.github/git-cliff-changelog.toml' --strip header --tag "${NEXT_TAG}" "${PREVIOUS_TAG}^.." \ + | sed "/## ${PREVIOUS_TAG#v}\$/,\$d" \ + | sed '$s/$/\n- - -/' > release-notes.md + + # Append release notes after the specified pattern in CHANGELOG.md + sed --in-place '0,/^- - -/!b; /^- - -/{ + a + r release-notes.md + }' CHANGELOG.md + rm release-notes.md + + - name: Set Git Configuration + shell: bash + run: | + git config --local user.name 'github-actions' + git config --local user.email '41898282+github-actions[bot]@users.noreply.github.com' + + - name: Push created commit and tag + shell: bash + run: | + # Stage, commit and tag the changelog + git add CHANGELOG.md + git commit --message "chore(version): ${NEXT_TAG}" + git tag --message "$(git show --no-patch --format=%s HEAD)" "${NEXT_TAG}" HEAD + git push + git push --tags diff --git a/.github/workflows/hotfix-branch-creation.yml b/.github/workflows/hotfix-branch-creation.yml new file mode 100644 index 000000000000..77a8bad6bc66 --- /dev/null +++ b/.github/workflows/hotfix-branch-creation.yml @@ -0,0 +1,38 @@ +name: Create hotfix branch + +on: + workflow_dispatch: + +jobs: + create_branch: + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + with: + fetch-depth: 0 + token: ${{ secrets.AUTO_RELEASE_PAT }} + + - name: Check if the input is valid tag + shell: bash + run: | + if [[ ${{github.ref}} =~ ^refs/tags/v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "::notice::${{github.ref}} is a valid tag." + else + echo "::error::${{github.ref}} is not a valid tag." + exit 1 + fi + + - name: Create hotfix branch + shell: bash + run: | + HOTFIX_BRANCH="hotfix-${GITHUB_REF#refs/tags/v}" + + if git switch --create "$HOTFIX_BRANCH"; then + git push origin "$HOTFIX_BRANCH" + echo "::notice::Created hotfix branch: $HOTFIX_BRANCH" + else + echo "::error::Failed to create hotfix branch" + exit 1 + fi \ No newline at end of file diff --git a/.github/workflows/release-new-version.yml b/.github/workflows/release-new-version.yml index 6b6362498538..4b2995174922 100644 --- a/.github/workflows/release-new-version.yml +++ b/.github/workflows/release-new-version.yml @@ -2,7 +2,7 @@ name: Release a new version on: schedule: - - cron: "30 14 * * 1-5" # Run workflow at 8 PM IST every Monday-Friday + - cron: "30 14 * * 0-4" # Run workflow at 8 PM IST every Sunday-Thursday workflow_dispatch: diff --git a/.github/workflows/validate-openapi-spec.yml b/.github/workflows/validate-openapi-spec.yml index 5c4c28518e67..530c59c9236d 100644 --- a/.github/workflows/validate-openapi-spec.yml +++ b/.github/workflows/validate-openapi-spec.yml @@ -16,14 +16,21 @@ jobs: name: Validate generated OpenAPI spec file runs-on: ubuntu-latest steps: - - name: Checkout PR - if: ${{ github.event_name == 'pull_request' }} + - name: Checkout PR from fork + if: ${{ (github.event_name == 'pull_request') && (github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name) }} uses: actions/checkout@v3 with: - # Checkout pull request branch instead of merge commit ref: ${{ github.event.pull_request.head.ref }} repository: ${{ github.event.pull_request.head.repo.full_name }} + - name: Checkout PR with token + if: ${{ (github.event_name == 'pull_request') && (github.event.pull_request.head.repo.full_name == github.event.pull_request.base.repo.full_name) }} + uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.ref }} + repository: ${{ github.event.pull_request.head.repo.full_name }} + token: ${{ secrets.AUTO_FILE_UPDATE_PAT }} + - name: Checkout merge group HEAD commit if: ${{ github.event_name == 'merge_group' }} uses: actions/checkout@v3 @@ -47,7 +54,21 @@ jobs: shell: bash run: swagger-cli validate ./openapi/openapi_spec.json + - name: Commit the JSON file if it is not up-to-date + # PR originated from same repository + if: ${{ (github.event_name == 'pull_request') && (github.event.pull_request.head.repo.full_name == github.event.pull_request.base.repo.full_name) }} + shell: bash + run: | + if ! git diff --quiet --exit-code -- openapi/openapi_spec.json ; then + git config --local user.name 'github-actions[bot]' + git config --local user.email '41898282+github-actions[bot]@users.noreply.github.com' + git add openapi/openapi_spec.json + git commit --message 'docs(openapi): re-generate OpenAPI specification' + git push + fi + - name: Fail check if the JSON file is not up-to-date + if: ${{ (github.event_name == 'merge_group') || ((github.event_name == 'pull_request') && (github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name)) }} shell: bash run: | if ! git diff --quiet --exit-code -- openapi/openapi_spec.json ; then diff --git a/CHANGELOG.md b/CHANGELOG.md index 65292e4fcaf3..9a3ef270b471 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,96 @@ All notable changes to HyperSwitch will be documented here. - - - +## 1.55.0 (2023-10-10) + +### Features + +- **connector:** + - [Multisafepay] Use connector_request_reference_id as reference to the connector ([#2503](https://github.com/juspay/hyperswitch/pull/2503)) ([`c34f1bf`](https://github.com/juspay/hyperswitch/commit/c34f1bf36ffb3a3533dd51ac87e7f66ab0dcce79)) + - [GlobalPayments] Introduce connector_request_reference_id for GlobalPayments ([#2519](https://github.com/juspay/hyperswitch/pull/2519)) ([`116139b`](https://github.com/juspay/hyperswitch/commit/116139ba7ae6878b7018068b0cb8303a8e8d1f7a)) + - [Airwallex] Use connector_request_reference_id as merchant reference id #2291 ([#2516](https://github.com/juspay/hyperswitch/pull/2516)) ([`6e89e41`](https://github.com/juspay/hyperswitch/commit/6e89e4103da4ecf6d7f06f7a9ec7da64eb493a6e)) +- **trace:** Add optional sampling behaviour for routes ([#2511](https://github.com/juspay/hyperswitch/pull/2511)) ([`ec51e48`](https://github.com/juspay/hyperswitch/commit/ec51e48402da63e1250328485095b8665d7eca65)) +- Gracefully shutdown drainer if redis goes down ([#2391](https://github.com/juspay/hyperswitch/pull/2391)) ([`2870af1`](https://github.com/juspay/hyperswitch/commit/2870af1286e897be0d40c014bc5742eafc6795db)) +- Kv for reverse lookup ([#2445](https://github.com/juspay/hyperswitch/pull/2445)) ([`13aaf96`](https://github.com/juspay/hyperswitch/commit/13aaf96db0f62dc7a706ba2ba230912ee7ef7a68)) +- Add x-hs-latency header for application overhead measurement ([#2486](https://github.com/juspay/hyperswitch/pull/2486)) ([`cf0db35`](https://github.com/juspay/hyperswitch/commit/cf0db35923d39caca9bf267b7d87a3f215884b66)) + +### Bug Fixes + +- **connector:** + - [Airwallex] convert expiry year to four digit ([#2527](https://github.com/juspay/hyperswitch/pull/2527)) ([`4b0fa12`](https://github.com/juspay/hyperswitch/commit/4b0fa1295ca8f4e611b65fbf2458c38b89303d3b)) + - [noon] add missing response status ([#2528](https://github.com/juspay/hyperswitch/pull/2528)) ([`808ee45`](https://github.com/juspay/hyperswitch/commit/808ee45556f90b1c1360a3edbffe9ba3603439d4)) + +### Refactors + +- **payment_methods:** Added mca_id in bank details ([#2495](https://github.com/juspay/hyperswitch/pull/2495)) ([`ac3c500`](https://github.com/juspay/hyperswitch/commit/ac3c5008f80172a575f2fb08b7a5e78016ce7595)) +- **test_utils:** Refactor `test_utils` crate and add `folder` support with updated documentation ([#2487](https://github.com/juspay/hyperswitch/pull/2487)) ([`6b52ac3`](https://github.com/juspay/hyperswitch/commit/6b52ac3d398d5a180c1dc67c5b53702ad01a0773)) + +### Miscellaneous Tasks + +- [GOCARDLESS] env changes for becs and sepa mandates ([#2535](https://github.com/juspay/hyperswitch/pull/2535)) ([`4f5a383`](https://github.com/juspay/hyperswitch/commit/4f5a383bab567a1b46b2d6990c0c23ed60f1201b)) + +**Full Changelog:** [`v1.54.0...v1.55.0`](https://github.com/juspay/hyperswitch/compare/v1.54.0...v1.55.0) + +- - - + + +## 1.54.0 (2023-10-09) + +### Features + +- **connector:** + - [Fiserv] update connector_response_reference_id in transformers ([#2489](https://github.com/juspay/hyperswitch/pull/2489)) ([`4eb7003`](https://github.com/juspay/hyperswitch/commit/4eb70034336e5ff42c9eea912d940ea04cae9326)) + - [Nuvei] Use "connector_request_reference_id" for as "attempt_id" to improve consistency in transmitting payment information ([#2493](https://github.com/juspay/hyperswitch/pull/2493)) ([`17393f5`](https://github.com/juspay/hyperswitch/commit/17393f5be3e9027fedf9466c6401754f3c4d6b99)) +- **kv:** Add kv wrapper for executing kv tasks ([#2384](https://github.com/juspay/hyperswitch/pull/2384)) ([`8b50997`](https://github.com/juspay/hyperswitch/commit/8b50997e56307507be101c562aa70d0a9b429137)) +- **process_tracker:** Make long standing payments failed ([#2380](https://github.com/juspay/hyperswitch/pull/2380)) ([`73dfc31`](https://github.com/juspay/hyperswitch/commit/73dfc31f9d16d2cf71de8433fb630bea941a7020)) + +### Bug Fixes + +- Add release feature to drianer ([#2507](https://github.com/juspay/hyperswitch/pull/2507)) ([`224b83c`](https://github.com/juspay/hyperswitch/commit/224b83c51d53fb1ca9ae11ff2f60b7b6cc807fc8)) + +### Refactors + +- Disable color in reports in json format ([#2509](https://github.com/juspay/hyperswitch/pull/2509)) ([`aa176c7`](https://github.com/juspay/hyperswitch/commit/aa176c7c5d79f68c8bd97a3248fd4d40e937a3ce)) + +### Miscellaneous Tasks + +- Address Rust 1.73 clippy lints ([#2474](https://github.com/juspay/hyperswitch/pull/2474)) ([`e02838e`](https://github.com/juspay/hyperswitch/commit/e02838eb5d3da97ef573926ded4a318ed24b6f1c)) + +**Full Changelog:** [`v1.53.0...v1.54.0`](https://github.com/juspay/hyperswitch/compare/v1.53.0...v1.54.0) + +- - - + + +## 1.53.0 (2023-10-09) + +### Features + +- **connector:** + - [Braintree] implement dispute webhook ([#2031](https://github.com/juspay/hyperswitch/pull/2031)) ([`eeccd10`](https://github.com/juspay/hyperswitch/commit/eeccd106ae569bd60011ed71495d7978998161f8)) + - [Paypal] Implement 3DS for Cards ([#2443](https://github.com/juspay/hyperswitch/pull/2443)) ([`d95a64d`](https://github.com/juspay/hyperswitch/commit/d95a64d6c9b870bdc38aa091cf9bf660b1ea404e)) + - [Cybersource] Use connector_response_reference_id as reference to merchant ([#2470](https://github.com/juspay/hyperswitch/pull/2470)) ([`a2dfc48`](https://github.com/juspay/hyperswitch/commit/a2dfc48318363db051f311ee7f911de0db0eb868)) + - [Coinbase] Add order id as the reference id ([#2469](https://github.com/juspay/hyperswitch/pull/2469)) ([`9c2fff5`](https://github.com/juspay/hyperswitch/commit/9c2fff5ab44cdd4f285b6d1437f37869b517963e)) + - [Multisafepay] Use transaction_id as reference to transaction ([#2451](https://github.com/juspay/hyperswitch/pull/2451)) ([`ba2efac`](https://github.com/juspay/hyperswitch/commit/ba2efac4fa2af22f81b0841350a334bc36e91022)) + +### Bug Fixes + +- Add startup config log to drainer ([#2482](https://github.com/juspay/hyperswitch/pull/2482)) ([`5038234`](https://github.com/juspay/hyperswitch/commit/503823408b782968fb59f6ff5d7df417b9aa7dbe)) +- Fetch data directly from DB in OLAP functions ([#2475](https://github.com/juspay/hyperswitch/pull/2475)) ([`12b5341`](https://github.com/juspay/hyperswitch/commit/12b534197276ccc4aa9575e6b518bcc50b597bee)) + +### Refactors + +- **connector:** [trustpay] refactor trustpay and handled variants errors ([#2484](https://github.com/juspay/hyperswitch/pull/2484)) ([`3f1e7c2`](https://github.com/juspay/hyperswitch/commit/3f1e7c2152a839a6fe69f60b906277ca831e7611)) +- **merchant_account:** Make `organization_id` as mandatory ([#2458](https://github.com/juspay/hyperswitch/pull/2458)) ([`53b4816`](https://github.com/juspay/hyperswitch/commit/53b4816d27fe7794cb482887ed17ddb4386bd2f7)) + +### Miscellaneous Tasks + +- Env changes for gocardless mandate ([#2485](https://github.com/juspay/hyperswitch/pull/2485)) ([`65ca5f1`](https://github.com/juspay/hyperswitch/commit/65ca5f12da54715e5db785d122e2ec9714147c68)) + +**Full Changelog:** [`v1.52.0...v1.53.0`](https://github.com/juspay/hyperswitch/compare/v1.52.0...v1.53.0) + +- - - + + ## 1.52.0 (2023-10-06) ### Features diff --git a/Cargo.lock b/Cargo.lock index 9e026101d1a1..b8290efee027 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4179,6 +4179,7 @@ version = "0.1.0" dependencies = [ "cargo_metadata 0.15.4", "config", + "error-stack", "gethostname", "once_cell", "opentelemetry", @@ -4792,6 +4793,7 @@ dependencies = [ "once_cell", "redis_interface", "ring", + "router_derive", "router_env", "serde", "serde_json", diff --git a/config/config.example.toml b/config/config.example.toml index e980ea4fffdb..56297fc20a44 100644 --- a/config/config.example.toml +++ b/config/config.example.toml @@ -28,6 +28,7 @@ port = 5432 # DB Port dbname = "hyperswitch_db" # Name of Database pool_size = 5 # Number of connections to keep open connection_timeout = 10 # Timeout for database connection in seconds +queue_strategy = "Fifo" # Add the queue strategy used by the database bb8 client # Replica SQL data store credentials [replica_database] @@ -38,6 +39,7 @@ port = 5432 # DB Port dbname = "hyperswitch_db" # Name of Database pool_size = 5 # Number of connections to keep open connection_timeout = 10 # Timeout for database connection in seconds +queue_strategy = "Fifo" # Add the queue strategy used by the database bb8 client # Redis credentials [redis] @@ -93,6 +95,7 @@ sampling_rate = 0.1 # decimal rate between 0.0 otel_exporter_otlp_endpoint = "http://localhost:4317" # endpoint to send metrics and traces to, can include port number otel_exporter_otlp_timeout = 5000 # timeout (in milliseconds) for sending metrics and traces use_xray_generator = false # Set this to true for AWS X-ray compatible traces +route_to_trace = [ "*/confirm" ] # This section provides some secret values. [secrets] @@ -318,9 +321,11 @@ square = {long_lived_token = false, payment_method = "card"} braintree = { long_lived_token = false, payment_method = "card" } gocardless = {long_lived_token = true, payment_method = "bank_debit"} -[temp_locker_disable_config] -trustpay = {payment_method = "card,bank_redirect,wallet"} -stripe = {payment_method = "card,bank_redirect,pay_later,wallet,bank_debit"} +[temp_locker_enable_config] +stripe = {payment_method = "bank_transfer"} +nuvei = {payment_method = "card"} +shift4 = {payment_method = "card"} +bluesnap = {payment_method = "card"} [dummy_connector] enabled = true # Whether dummy connector is enabled or not @@ -347,6 +352,8 @@ card.credit = {connector_list = "stripe,adyen"} # Mandate supported payment wallet.paypal = {connector_list = "adyen"} # Mandate supported payment method type and connector for wallets pay_later.klarna = {connector_list = "adyen"} # Mandate supported payment method type and connector for pay_later bank_debit.ach = { connector_list = "gocardless"} # Mandate supported payment method type and connector for bank_debit +bank_debit.becs = { connector_list = "gocardless"} # Mandate supported payment method type and connector for bank_debit +bank_debit.sepa = { connector_list = "gocardless"} # Mandate supported payment method type and connector for bank_debit # Required fields info used while listing the payment_method_data [required_fields.pay_later] # payment_method = "pay_later" @@ -422,4 +429,4 @@ supported_connectors = "braintree" apple_pay_ppc = "APPLE_PAY_PAYMENT_PROCESSING_CERTIFICATE" #Payment Processing Certificate provided by Apple Pay (https://developer.apple.com/) Certificates, Identifiers & Profiles > Apple Pay Payment Processing Certificate apple_pay_ppc_key = "APPLE_PAY_PAYMENT_PROCESSING_CERTIFICATE_KEY" #Private key generate by Elliptic-curve prime256v1 curve apple_pay_merchant_cert = "APPLE_PAY_MERCHNAT_CERTIFICATE" #Merchant Certificate provided by Apple Pay (https://developer.apple.com/) Certificates, Identifiers & Profiles > Apple Pay Merchant Identity Certificate -apple_pay_merchant_cert_key = "APPLE_PAY_MERCHNAT_CERTIFICATE_KEY" #Private key generate by RSA:2048 algorithm \ No newline at end of file +apple_pay_merchant_cert_key = "APPLE_PAY_MERCHNAT_CERTIFICATE_KEY" #Private key generate by RSA:2048 algorithm diff --git a/config/development.toml b/config/development.toml index 395b7f2eea03..6d5d02c3bb01 100644 --- a/config/development.toml +++ b/config/development.toml @@ -377,9 +377,11 @@ braintree = { long_lived_token = false, payment_method = "card" } payme = {long_lived_token = false, payment_method = "card"} gocardless = {long_lived_token = true, payment_method = "bank_debit"} -[temp_locker_disable_config] -trustpay = {payment_method = "card,bank_redirect,wallet"} -stripe = {payment_method = "card,bank_redirect,pay_later,wallet,bank_debit"} +[temp_locker_enable_config] +stripe = {payment_method = "bank_transfer"} +nuvei = {payment_method = "card"} +shift4 = {payment_method = "card"} +bluesnap = {payment_method = "card"} [connector_customer] connector_list = "gocardless,stax,stripe" @@ -419,6 +421,8 @@ wallet.paypal = { connector_list = "adyen" } card.credit = { connector_list = "stripe,adyen,authorizedotnet,globalpay,worldpay,multisafepay,nmi,nexinets,noon" } card.debit = { connector_list = "stripe,adyen,authorizedotnet,globalpay,worldpay,multisafepay,nmi,nexinets,noon" } bank_debit.ach = { connector_list = "gocardless"} +bank_debit.becs = { connector_list = "gocardless"} +bank_debit.sepa = { connector_list = "gocardless"} [connector_request_reference_id_config] merchant_ids_send_payment_id_as_connector_request_id = [] @@ -437,4 +441,4 @@ apple_pay_merchant_cert_key = "APPLE_PAY_MERCHNAT_CERTIFICATE_KEY" [lock_settings] redis_lock_expiry_seconds = 180 # 3 * 60 seconds -delay_between_retries_in_milliseconds = 500 +delay_between_retries_in_milliseconds = 500 \ No newline at end of file diff --git a/config/docker_compose.toml b/config/docker_compose.toml index b1483327ee0c..983abd7cc3c8 100644 --- a/config/docker_compose.toml +++ b/config/docker_compose.toml @@ -208,9 +208,11 @@ square = {long_lived_token = false, payment_method = "card"} braintree = { long_lived_token = false, payment_method = "card" } gocardless = {long_lived_token = true, payment_method = "bank_debit"} -[temp_locker_disable_config] -trustpay = {payment_method = "card,bank_redirect,wallet"} -stripe = {payment_method = "card,bank_redirect,pay_later,wallet,bank_debit"} +[temp_locker_enable_config] +stripe = {payment_method = "bank_transfer"} +nuvei = {payment_method = "card"} +shift4 = {payment_method = "card"} +bluesnap = {payment_method = "card"} [dummy_connector] enabled = true @@ -297,6 +299,8 @@ wallet.paypal = {connector_list = "adyen"} card.credit = {connector_list = "stripe,adyen,authorizedotnet,globalpay,worldpay,multisafepay,nmi,nexinets,noon"} card.debit = {connector_list = "stripe,adyen,authorizedotnet,globalpay,worldpay,multisafepay,nmi,nexinets,noon"} bank_debit.ach = { connector_list = "gocardless"} +bank_debit.becs = { connector_list = "gocardless"} +bank_debit.sepa = { connector_list = "gocardless"} [connector_customer] connector_list = "gocardless,stax,stripe" diff --git a/crates/api_models/src/payment_methods.rs b/crates/api_models/src/payment_methods.rs index 067a3a341402..eb15f0e3bc3a 100644 --- a/crates/api_models/src/payment_methods.rs +++ b/crates/api_models/src/payment_methods.rs @@ -174,6 +174,7 @@ pub struct PaymentMethodDataBankCreds { pub struct BankAccountConnectorDetails { pub connector: String, pub account_id: String, + pub mca_id: String, pub access_token: BankAccountAccessCreds, } diff --git a/crates/api_models/src/payments.rs b/crates/api_models/src/payments.rs index 095e30f38219..01edef87a67a 100644 --- a/crates/api_models/src/payments.rs +++ b/crates/api_models/src/payments.rs @@ -307,6 +307,7 @@ pub struct PaymentsRequest { #[derive(Default, Debug, Clone, Copy)] pub struct HeaderPayload { pub payment_confirm_source: Option, + pub x_hs_latency: Option, } #[derive( diff --git a/crates/common_utils/src/consts.rs b/crates/common_utils/src/consts.rs index ef688f728b09..2f517295ae48 100644 --- a/crates/common_utils/src/consts.rs +++ b/crates/common_utils/src/consts.rs @@ -26,3 +26,6 @@ pub const PAYMENTS_LIST_MAX_LIMIT_V2: u32 = 20; /// surcharge percentage maximum precision length pub const SURCHARGE_PERCENTAGE_PRECISION_LENGTH: u8 = 2; + +/// Header Key for application overhead of a request +pub const X_HS_LATENCY: &str = "x-hs-latency"; diff --git a/crates/diesel_models/src/kv.rs b/crates/diesel_models/src/kv.rs index 5804107d6551..93d045bd71a0 100644 --- a/crates/diesel_models/src/kv.rs +++ b/crates/diesel_models/src/kv.rs @@ -8,6 +8,7 @@ use crate::{ payment_attempt::{PaymentAttempt, PaymentAttemptNew, PaymentAttemptUpdate}, payment_intent::{PaymentIntent, PaymentIntentNew, PaymentIntentUpdate}, refund::{Refund, RefundNew, RefundUpdate}, + reverse_lookup::ReverseLookupNew, }; #[derive(Debug, Serialize, Deserialize)] @@ -43,6 +44,7 @@ pub enum Insertable { Refund(RefundNew), ConnectorResponse(ConnectorResponseNew), Address(Box), + ReverseLookUp(ReverseLookupNew), } #[derive(Debug, Serialize, Deserialize)] diff --git a/crates/diesel_models/src/reverse_lookup.rs b/crates/diesel_models/src/reverse_lookup.rs index a72b23accc67..6483b7c78cd6 100644 --- a/crates/diesel_models/src/reverse_lookup.rs +++ b/crates/diesel_models/src/reverse_lookup.rs @@ -22,7 +22,14 @@ pub struct ReverseLookup { } #[derive( - Clone, Debug, Insertable, router_derive::DebugAsDisplay, Eq, PartialEq, serde::Serialize, + Clone, + Debug, + Insertable, + router_derive::DebugAsDisplay, + Eq, + PartialEq, + serde::Serialize, + serde::Deserialize, )] #[diesel(table_name = reverse_lookup)] pub struct ReverseLookupNew { diff --git a/crates/drainer/Cargo.toml b/crates/drainer/Cargo.toml index 45186b84ca9d..2b0ade029304 100644 --- a/crates/drainer/Cargo.toml +++ b/crates/drainer/Cargo.toml @@ -8,6 +8,7 @@ readme = "README.md" license.workspace = true [features] +release = ["kms","vergen"] kms = ["external_services/kms"] vergen = ["router_env/vergen"] diff --git a/crates/drainer/src/lib.rs b/crates/drainer/src/lib.rs index fd3fa47d8069..e09c56540572 100644 --- a/crates/drainer/src/lib.rs +++ b/crates/drainer/src/lib.rs @@ -10,7 +10,7 @@ use std::sync::{atomic, Arc}; use common_utils::signals::get_allowed_signals; use diesel_models::kv; use error_stack::{IntoReport, ResultExt}; -use tokio::sync::mpsc; +use tokio::sync::{mpsc, oneshot}; use crate::{connection::pg_connection, services::Store}; @@ -36,9 +36,18 @@ pub async fn start_drainer( "Failed while getting allowed signals".to_string(), ))?; + let (redis_error_tx, redis_error_rx) = oneshot::channel(); let (tx, mut rx) = mpsc::channel(1); let handle = signal.handle(); - let task_handle = tokio::spawn(common_utils::signals::signal_handler(signal, tx)); + let task_handle = tokio::spawn(common_utils::signals::signal_handler(signal, tx.clone())); + + let redis_conn_clone = store.redis_conn.clone(); + + // Spawn a task to monitor if redis is down or not + tokio::spawn(async move { redis_conn_clone.on_error(redis_error_tx).await }); + + //Spawns a task to send shutdown signal if redis goes down + tokio::spawn(redis_error_receiver(redis_error_rx, tx)); let active_tasks = Arc::new(atomic::AtomicU64::new(0)); 'event: loop { @@ -90,6 +99,20 @@ pub async fn start_drainer( Ok(()) } +pub async fn redis_error_receiver(rx: oneshot::Receiver<()>, shutdown_channel: mpsc::Sender<()>) { + match rx.await { + Ok(_) => { + logger::error!("The redis server failed "); + let _ = shutdown_channel.send(()).await.map_err(|err| { + logger::error!("Failed to send signal to the shutdown channel {err}") + }); + } + Err(err) => { + logger::error!("Channel receiver error{err}"); + } + } +} + async fn drainer_handler( store: Arc, stream_index: u8, @@ -165,6 +188,7 @@ async fn drainer( let payment_intent = "payment_intent"; let payment_attempt = "payment_attempt"; let refund = "refund"; + let reverse_lookup = "reverse_lookup"; let connector_response = "connector_response"; let address = "address"; match db_op { @@ -199,6 +223,13 @@ async fn drainer( kv::Insertable::Address(addr) => { macro_util::handle_resp!(addr.insert(&conn).await, insert_op, address) } + kv::Insertable::ReverseLookUp(rev) => { + macro_util::handle_resp!( + rev.insert(&conn).await, + insert_op, + reverse_lookup + ) + } } }) .await; diff --git a/crates/redis_interface/src/commands.rs b/crates/redis_interface/src/commands.rs index 939586116fcc..d53fd1625fe4 100644 --- a/crates/redis_interface/src/commands.rs +++ b/crates/redis_interface/src/commands.rs @@ -65,6 +65,22 @@ impl super::RedisConnectionPool { .change_context(errors::RedisError::SetFailed) } + #[instrument(level = "DEBUG", skip(self))] + pub async fn serialize_and_set_key_if_not_exist( + &self, + key: &str, + value: V, + ttl: Option, + ) -> CustomResult + where + V: serde::Serialize + Debug, + { + let serialized = Encode::::encode_to_vec(&value) + .change_context(errors::RedisError::JsonSerializationFailed)?; + self.set_key_if_not_exists_with_expiry(key, serialized.as_slice(), ttl) + .await + } + #[instrument(level = "DEBUG", skip(self))] pub async fn serialize_and_set_key( &self, @@ -232,6 +248,7 @@ impl super::RedisConnectionPool { &self, key: &str, values: V, + ttl: Option, ) -> CustomResult<(), errors::RedisError> where V: TryInto + Debug + Send + Sync, @@ -246,7 +263,9 @@ impl super::RedisConnectionPool { // setting expiry for the key output - .async_and_then(|_| self.set_expiry(key, self.config.default_hash_ttl.into())) + .async_and_then(|_| { + self.set_expiry(key, ttl.unwrap_or(self.config.default_hash_ttl).into()) + }) .await } @@ -256,6 +275,7 @@ impl super::RedisConnectionPool { key: &str, field: &str, value: V, + ttl: Option, ) -> CustomResult where V: TryInto + Debug + Send + Sync, @@ -270,7 +290,7 @@ impl super::RedisConnectionPool { output .async_and_then(|inner| async { - self.set_expiry(key, self.config.default_hash_ttl.into()) + self.set_expiry(key, ttl.unwrap_or(self.config.default_hash_ttl).into()) .await?; Ok(inner) }) @@ -283,6 +303,7 @@ impl super::RedisConnectionPool { key: &str, field: &str, value: V, + ttl: Option, ) -> CustomResult where V: serde::Serialize + Debug, @@ -290,7 +311,7 @@ impl super::RedisConnectionPool { let serialized = Encode::::encode_to_vec(&value) .change_context(errors::RedisError::JsonSerializationFailed)?; - self.set_hash_field_if_not_exist(key, field, serialized.as_slice()) + self.set_hash_field_if_not_exist(key, field, serialized.as_slice(), ttl) .await } @@ -339,6 +360,7 @@ impl super::RedisConnectionPool { &self, kv: &[(&str, V)], field: &str, + ttl: Option, ) -> CustomResult, errors::RedisError> where V: serde::Serialize + Debug, @@ -346,7 +368,7 @@ impl super::RedisConnectionPool { let mut hsetnx: Vec = Vec::with_capacity(kv.len()); for (key, val) in kv { hsetnx.push( - self.serialize_and_set_hash_field_if_not_exist(key, field, val) + self.serialize_and_set_hash_field_if_not_exist(key, field, val, ttl) .await?, ); } diff --git a/crates/redis_interface/src/errors.rs b/crates/redis_interface/src/errors.rs index 4ea88638ac78..213fb799892e 100644 --- a/crates/redis_interface/src/errors.rs +++ b/crates/redis_interface/src/errors.rs @@ -62,4 +62,6 @@ pub enum RedisError { PublishError, #[error("Failed while receiving message from publisher")] OnMessageError, + #[error("Got an unknown result from redis")] + UnknownResult, } diff --git a/crates/router/src/compatibility/wrap.rs b/crates/router/src/compatibility/wrap.rs index 281d323cc517..edc56fcee731 100644 --- a/crates/router/src/compatibility/wrap.rs +++ b/crates/router/src/compatibility/wrap.rs @@ -85,7 +85,7 @@ where let response = S::try_from(response); match response { Ok(response) => match serde_json::to_string(&response) { - Ok(res) => api::http_response_json_with_headers(res, headers), + Ok(res) => api::http_response_json_with_headers(res, headers, None), Err(_) => api::http_response_err( r#"{ "error": { diff --git a/crates/router/src/configs/defaults.rs b/crates/router/src/configs/defaults.rs index ec06a6d7078f..3bb1c31d180b 100644 --- a/crates/router/src/configs/defaults.rs +++ b/crates/router/src/configs/defaults.rs @@ -29,6 +29,7 @@ impl Default for super::settings::Database { dbname: String::new(), pool_size: 5, connection_timeout: 10, + queue_strategy: Default::default(), } } } diff --git a/crates/router/src/configs/kms.rs b/crates/router/src/configs/kms.rs index 0491fb61fc6e..317ad0608b49 100644 --- a/crates/router/src/configs/kms.rs +++ b/crates/router/src/configs/kms.rs @@ -61,6 +61,7 @@ impl KmsDecrypt for settings::Database { password: self.password.decrypt_inner(kms_client).await?.into(), pool_size: self.pool_size, connection_timeout: self.connection_timeout, + queue_strategy: self.queue_strategy.into(), }) } } diff --git a/crates/router/src/configs/settings.rs b/crates/router/src/configs/settings.rs index fd4a398fd568..a2feb55e86a2 100644 --- a/crates/router/src/configs/settings.rs +++ b/crates/router/src/configs/settings.rs @@ -99,7 +99,7 @@ pub struct Settings { pub multiple_api_version_supported_connectors: MultipleApiVersionSupportedConnectors, pub applepay_merchant_configs: ApplepayMerchantConfigs, pub lock_settings: LockSettings, - pub temp_locker_disable_config: TempLockerDisableConfig, + pub temp_locker_enable_config: TempLockerEnableConfig, } #[derive(Debug, Deserialize, Clone, Default)] @@ -123,7 +123,7 @@ pub struct TokenizationConfig(pub HashMap); #[derive(Debug, Deserialize, Clone, Default)] #[serde(transparent)] -pub struct TempLockerDisableConfig(pub HashMap); +pub struct TempLockerEnableConfig(pub HashMap); #[derive(Debug, Deserialize, Clone, Default)] pub struct ConnectorCustomer { @@ -216,7 +216,7 @@ pub struct PaymentMethodTokenFilter { } #[derive(Debug, Deserialize, Clone, Default)] -pub struct TempLockerDisablePaymentMethodFilter { +pub struct TempLockerEnablePaymentMethodFilter { #[serde(deserialize_with = "pm_deser")] pub payment_method: HashSet, } @@ -464,6 +464,24 @@ pub struct Database { pub dbname: String, pub pool_size: u32, pub connection_timeout: u64, + pub queue_strategy: QueueStrategy, +} + +#[derive(Debug, Deserialize, Clone, Default)] +#[serde(rename_all = "PascalCase")] +pub enum QueueStrategy { + #[default] + Fifo, + Lifo, +} + +impl From for bb8::QueueStrategy { + fn from(value: QueueStrategy) -> Self { + match value { + QueueStrategy::Fifo => Self::Fifo, + QueueStrategy::Lifo => Self::Lifo, + } + } } #[cfg(not(feature = "kms"))] @@ -477,6 +495,7 @@ impl Into for Database { dbname: self.dbname, pool_size: self.pool_size, connection_timeout: self.connection_timeout, + queue_strategy: self.queue_strategy.into(), } } } @@ -704,9 +723,11 @@ impl Settings { .try_parsing(true) .separator("__") .list_separator(",") + .with_list_parse_key("log.telemetry.route_to_trace") .with_list_parse_key("redis.cluster_urls") .with_list_parse_key("connectors.supported.wallets") .with_list_parse_key("connector_request_reference_id_config.merchant_ids_send_payment_id_as_connector_request_id"), + ) .build()?; diff --git a/crates/router/src/connector/airwallex/transformers.rs b/crates/router/src/connector/airwallex/transformers.rs index ecdddfb36726..3343f41c554e 100644 --- a/crates/router/src/connector/airwallex/transformers.rs +++ b/crates/router/src/connector/airwallex/transformers.rs @@ -6,7 +6,7 @@ use url::Url; use uuid::Uuid; use crate::{ - connector::utils, + connector::utils::{self, CardData}, core::errors, pii::Secret, services, @@ -48,7 +48,7 @@ impl TryFrom<&types::PaymentsInitRouterData> for AirwallexIntentRequest { request_id: Uuid::new_v4().to_string(), amount: utils::to_currency_base_unit(item.request.amount, item.request.currency)?, currency: item.request.currency, - merchant_order_id: item.payment_id.clone(), + merchant_order_id: item.connector_request_reference_id.clone(), }) } } @@ -140,9 +140,9 @@ impl TryFrom<&types::PaymentsAuthorizeRouterData> for AirwallexPaymentsRequest { })); Ok(AirwallexPaymentMethod::Card(AirwallexCard { card: AirwallexCardDetails { - number: ccard.card_number, + number: ccard.card_number.clone(), expiry_month: ccard.card_exp_month.clone(), - expiry_year: ccard.card_exp_year.clone(), + expiry_year: ccard.get_expiry_year_4_digit(), cvc: ccard.card_cvc, }, payment_method_type: AirwallexPaymentType::Card, diff --git a/crates/router/src/connector/checkout.rs b/crates/router/src/connector/checkout.rs index fb0aa8208925..52c4351287c0 100644 --- a/crates/router/src/connector/checkout.rs +++ b/crates/router/src/connector/checkout.rs @@ -110,7 +110,7 @@ impl ConnectorCommon for Checkout { }; router_env::logger::info!(error_response=?response); - let errors_list = response.error_codes.clone().unwrap_or(vec![]); + let errors_list = response.error_codes.clone().unwrap_or_default(); let option_error_code_message = conn_utils::get_error_code_error_message_based_on_priority( self.clone(), errors_list diff --git a/crates/router/src/connector/cybersource.rs b/crates/router/src/connector/cybersource.rs index 11f76ec33131..d1ad36b26d1e 100644 --- a/crates/router/src/connector/cybersource.rs +++ b/crates/router/src/connector/cybersource.rs @@ -101,7 +101,7 @@ impl ConnectorCommon for Cybersource { .response .parse_struct("Cybersource ErrorResponse") .change_context(errors::ConnectorError::ResponseDeserializationFailed)?; - let details = response.details.unwrap_or(vec![]); + let details = response.details.unwrap_or_default(); let connector_reason = details .iter() .map(|det| format!("{} : {}", det.field, det.reason)) diff --git a/crates/router/src/connector/fiserv/transformers.rs b/crates/router/src/connector/fiserv/transformers.rs index 6c6c346dcccd..c9c2f0c4087a 100644 --- a/crates/router/src/connector/fiserv/transformers.rs +++ b/crates/router/src/connector/fiserv/transformers.rs @@ -315,7 +315,9 @@ impl mandate_reference: None, connector_metadata: None, network_txn_id: None, - connector_response_reference_id: None, + connector_response_reference_id: Some( + gateway_resp.transaction_processing_details.order_id, + ), }), ..item.data }) @@ -350,7 +352,13 @@ impl TryFrom for GlobalpayPaymentsRequest { account_name, amount: Some(item.request.amount.to_string()), currency: item.request.currency.to_string(), - reference: item.attempt_id.to_string(), + reference: item.connector_request_reference_id.to_string(), country: item.get_billing_country()?, capture_mode: Some(requests::CaptureMode::from(item.request.capture_method)), payment_method: requests::PaymentMethod { diff --git a/crates/router/src/connector/gocardless/transformers.rs b/crates/router/src/connector/gocardless/transformers.rs index 8187c5ccc49d..c5322ccf7fc6 100644 --- a/crates/router/src/connector/gocardless/transformers.rs +++ b/crates/router/src/connector/gocardless/transformers.rs @@ -582,7 +582,7 @@ impl mandate_reference, network_txn_id: None, }), - status: enums::AttemptStatus::Pending, + status: enums::AttemptStatus::Charged, ..item.data }) } diff --git a/crates/router/src/connector/multisafepay.rs b/crates/router/src/connector/multisafepay.rs index 1629f2ab36d9..120ea23d7ca6 100644 --- a/crates/router/src/connector/multisafepay.rs +++ b/crates/router/src/connector/multisafepay.rs @@ -165,7 +165,7 @@ impl ConnectorIntegration for MultisafepayPaymentsReques Ok(Self { payment_type, gateway, - order_id: item.payment_id.to_string(), + order_id: item.connector_request_reference_id.to_string(), currency: item.request.currency.to_string(), amount: item.request.amount, description, diff --git a/crates/router/src/connector/noon/transformers.rs b/crates/router/src/connector/noon/transformers.rs index 3e584e204ae1..5300525b7cbd 100644 --- a/crates/router/src/connector/noon/transformers.rs +++ b/crates/router/src/connector/noon/transformers.rs @@ -326,6 +326,8 @@ pub enum NoonPaymentStatus { Failed, #[default] Pending, + Expired, + Rejected, } impl From for enums::AttemptStatus { @@ -334,11 +336,11 @@ impl From for enums::AttemptStatus { NoonPaymentStatus::Authorized => Self::Authorized, NoonPaymentStatus::Captured | NoonPaymentStatus::PartiallyCaptured => Self::Charged, NoonPaymentStatus::Reversed => Self::Voided, - NoonPaymentStatus::Cancelled => Self::AuthenticationFailed, + NoonPaymentStatus::Cancelled | NoonPaymentStatus::Expired => Self::AuthenticationFailed, NoonPaymentStatus::ThreeDsEnrollInitiated | NoonPaymentStatus::ThreeDsEnrollChecked => { Self::AuthenticationPending } - NoonPaymentStatus::Failed => Self::Failure, + NoonPaymentStatus::Failed | NoonPaymentStatus::Rejected => Self::Failure, NoonPaymentStatus::Pending => Self::Pending, } } diff --git a/crates/router/src/connector/nuvei/transformers.rs b/crates/router/src/connector/nuvei/transformers.rs index cf7d480e0c18..2fd1a9c272f1 100644 --- a/crates/router/src/connector/nuvei/transformers.rs +++ b/crates/router/src/connector/nuvei/transformers.rs @@ -384,7 +384,7 @@ impl TryFrom<&types::PaymentsAuthorizeSessionTokenRouterData> for NuveiSessionRe let connector_meta: NuveiAuthType = NuveiAuthType::try_from(&item.connector_auth_type)?; let merchant_id = connector_meta.merchant_id; let merchant_site_id = connector_meta.merchant_site_id; - let client_request_id = item.attempt_id.clone(); + let client_request_id = item.connector_request_reference_id.clone(); let time_stamp = date_time::DateTime::::from(date_time::now()); let merchant_secret = connector_meta.merchant_secret; Ok(Self { @@ -737,7 +737,7 @@ impl amount: utils::to_currency_base_unit(item.request.amount, item.request.currency)?, currency: item.request.currency, connector_auth_type: item.connector_auth_type.clone(), - client_request_id: item.attempt_id.clone(), + client_request_id: item.connector_request_reference_id.clone(), session_token: data.1, capture_method: item.request.capture_method, ..Default::default() @@ -914,7 +914,7 @@ impl TryFrom<(&types::PaymentsCompleteAuthorizeRouterData, String)> for NuveiPay amount: utils::to_currency_base_unit(item.request.amount, item.request.currency)?, currency: item.request.currency, connector_auth_type: item.connector_auth_type.clone(), - client_request_id: item.attempt_id.clone(), + client_request_id: item.connector_request_reference_id.clone(), session_token: data.1, capture_method: item.request.capture_method, ..Default::default() @@ -1018,7 +1018,7 @@ impl TryFrom<&types::PaymentsCaptureRouterData> for NuveiPaymentFlowRequest { type Error = error_stack::Report; fn try_from(item: &types::PaymentsCaptureRouterData) -> Result { Self::try_from(NuveiPaymentRequestData { - client_request_id: item.attempt_id.clone(), + client_request_id: item.connector_request_reference_id.clone(), connector_auth_type: item.connector_auth_type.clone(), amount: utils::to_currency_base_unit( item.request.amount_to_capture, @@ -1034,7 +1034,7 @@ impl TryFrom<&types::RefundExecuteRouterData> for NuveiPaymentFlowRequest { type Error = error_stack::Report; fn try_from(item: &types::RefundExecuteRouterData) -> Result { Self::try_from(NuveiPaymentRequestData { - client_request_id: item.attempt_id.clone(), + client_request_id: item.connector_request_reference_id.clone(), connector_auth_type: item.connector_auth_type.clone(), amount: utils::to_currency_base_unit( item.request.refund_amount, @@ -1061,7 +1061,7 @@ impl TryFrom<&types::PaymentsCancelRouterData> for NuveiPaymentFlowRequest { type Error = error_stack::Report; fn try_from(item: &types::PaymentsCancelRouterData) -> Result { Self::try_from(NuveiPaymentRequestData { - client_request_id: item.attempt_id.clone(), + client_request_id: item.connector_request_reference_id.clone(), connector_auth_type: item.connector_auth_type.clone(), amount: utils::to_currency_base_unit( item.request.get_amount()?, diff --git a/crates/router/src/connector/paypal.rs b/crates/router/src/connector/paypal.rs index 9ca418aa04a9..1faee2a16b1b 100644 --- a/crates/router/src/connector/paypal.rs +++ b/crates/router/src/connector/paypal.rs @@ -1,5 +1,5 @@ pub mod transformers; -use std::fmt::Debug; +use std::fmt::{Debug, Write}; use base64::Engine; use common_utils::ext_traits::ByteSliceExt; @@ -169,12 +169,26 @@ impl ConnectorCommon for Paypal { .parse_struct("Paypal ErrorResponse") .change_context(errors::ConnectorError::ResponseDeserializationFailed)?; - let error_reason = response.details.map(|error_details| { - error_details - .iter() - .map(|error| format!("description - {} ; ", error.description)) - .collect::() - }); + let error_reason = response + .details + .map(|error_details| { + error_details + .iter() + .try_fold::<_, _, CustomResult<_, errors::ConnectorError>>( + String::new(), + |mut acc, error| { + write!(acc, "description - {} ;", error.description) + .into_report() + .change_context( + errors::ConnectorError::ResponseDeserializationFailed, + ) + .attach_printable("Failed to concatenate error details") + .map(|_| acc) + }, + ) + }) + .transpose()?; + Ok(ErrorResponse { status_code: res.status_code, code: response.name, diff --git a/crates/router/src/connector/paypal/transformers.rs b/crates/router/src/connector/paypal/transformers.rs index 932cf67addd4..78bd75efe143 100644 --- a/crates/router/src/connector/paypal/transformers.rs +++ b/crates/router/src/connector/paypal/transformers.rs @@ -1655,7 +1655,7 @@ fn get_headers( key: &'static str, ) -> CustomResult { let header_value = header - .get(key.clone()) + .get(key) .map(|value| value.to_str()) .ok_or(errors::ConnectorError::MissingRequiredField { field_name: key })? .into_report() diff --git a/crates/router/src/consts.rs b/crates/router/src/consts.rs index 228d02e1ddac..89e4e864d180 100644 --- a/crates/router/src/consts.rs +++ b/crates/router/src/consts.rs @@ -13,6 +13,8 @@ pub(crate) const ALPHABETS: [char; 62] = [ pub const REQUEST_TIME_OUT: u64 = 30; pub const REQUEST_TIMEOUT_ERROR_CODE: &str = "TIMEOUT"; pub const REQUEST_TIMEOUT_ERROR_MESSAGE: &str = "Connector did not respond in specified time"; +pub const REQUEST_TIMEOUT_ERROR_MESSAGE_FROM_PSYNC: &str = + "This Payment has been moved to failed as there is no response from the connector"; ///Payment intent fulfillment default timeout (in seconds) pub const DEFAULT_FULFILLMENT_TIME: i64 = 15 * 60; @@ -44,3 +46,6 @@ pub(crate) const QR_IMAGE_DATA_SOURCE_STRING: &str = "data:image/png;base64"; pub(crate) const MERCHANT_ID_FIELD_EXTENSION_ID: &str = "1.2.840.113635.100.6.32"; pub(crate) const METRICS_HOST_TAG_NAME: &str = "host"; + +// TTL for KV setup +pub(crate) const KV_TTL: u32 = 300; diff --git a/crates/router/src/core/payment_methods/cards.rs b/crates/router/src/core/payment_methods/cards.rs index 4177a8d3627b..deccf98d83e5 100644 --- a/crates/router/src/core/payment_methods/cards.rs +++ b/crates/router/src/core/payment_methods/cards.rs @@ -977,9 +977,7 @@ pub async fn list_payment_methods( .0 .get(&payment_method_type) .map(|required_fields_hm_for_each_connector| { - required_fields_hm - .entry(payment_method) - .or_insert(HashMap::new()); + required_fields_hm.entry(payment_method).or_default(); required_fields_hm_for_each_connector .fields .get(&connector_variant) diff --git a/crates/router/src/core/payments.rs b/crates/router/src/core/payments.rs index cc84a13616da..d65e53c95ba9 100644 --- a/crates/router/src/core/payments.rs +++ b/crates/router/src/core/payments.rs @@ -66,7 +66,13 @@ pub async fn payments_operation_core( call_connector_action: CallConnectorAction, auth_flow: services::AuthFlow, header_payload: HeaderPayload, -) -> RouterResult<(PaymentData, Req, Option, Option)> +) -> RouterResult<( + PaymentData, + Req, + Option, + Option, + Option, +)> where F: Send + Clone + Sync, Req: Authenticate, @@ -158,7 +164,7 @@ where .await?; let mut connector_http_status_code = None; - + let mut external_latency = None; if let Some(connector_details) = connector { payment_data = match connector_details { api::ConnectorCallType::Single(connector) => { @@ -180,6 +186,7 @@ where let operation = Box::new(PaymentResponse); let db = &*state.store; connector_http_status_code = router_data.connector_http_status_code; + external_latency = router_data.external_latency; //add connector http status code metrics add_connector_http_status_code_metrics(connector_http_status_code); operation @@ -237,7 +244,13 @@ where .await?; } - Ok((payment_data, req, customer, connector_http_status_code)) + Ok(( + payment_data, + req, + customer, + connector_http_status_code, + external_latency, + )) } #[allow(clippy::too_many_arguments)] @@ -269,7 +282,7 @@ where // To perform router related operation for PaymentResponse PaymentResponse: Operation, { - let (payment_data, req, customer, connector_http_status_code) = + let (payment_data, req, customer, connector_http_status_code, external_latency) = payments_operation_core::<_, _, _, _, Ctx>( &state, merchant_account, @@ -291,6 +304,8 @@ where operation, &state.conf.connector_request_reference_id_config, connector_http_status_code, + external_latency, + header_payload.x_hs_latency, ) } diff --git a/crates/router/src/core/payments/customers.rs b/crates/router/src/core/payments/customers.rs index 5bfe7fbe1cfe..ff56eaf46ebf 100644 --- a/crates/router/src/core/payments/customers.rs +++ b/crates/router/src/core/payments/customers.rs @@ -121,7 +121,7 @@ pub async fn update_connector_customer_in_customers( .and_then(|customer| customer.connector_customer.as_ref()) .and_then(|connector_customer| connector_customer.as_object()) .map(ToOwned::to_owned) - .unwrap_or(serde_json::Map::new()); + .unwrap_or_default(); let updated_connector_customer_map = connector_customer_id.as_ref().map(|connector_customer_id| { diff --git a/crates/router/src/core/payments/helpers.rs b/crates/router/src/core/payments/helpers.rs index 9a0896665582..143b9923958c 100644 --- a/crates/router/src/core/payments/helpers.rs +++ b/crates/router/src/core/payments/helpers.rs @@ -31,7 +31,7 @@ use super::{ CustomerDetails, PaymentData, }; use crate::{ - configs::settings::{ConnectorRequestReferenceIdConfig, Server, TempLockerDisableConfig}, + configs::settings::{ConnectorRequestReferenceIdConfig, Server, TempLockerEnableConfig}, connector, consts::{self, BASE64_ENGINE}, core::{ @@ -1480,7 +1480,7 @@ pub async fn store_payment_method_data_in_vault( payment_method_data: &api::PaymentMethodData, ) -> RouterResult> { if should_store_payment_method_data_in_vault( - &state.conf.temp_locker_disable_config, + &state.conf.temp_locker_enable_config, payment_attempt.connector.clone(), payment_method, ) { @@ -1499,18 +1499,17 @@ pub async fn store_payment_method_data_in_vault( Ok(None) } pub fn should_store_payment_method_data_in_vault( - temp_locker_disable_config: &TempLockerDisableConfig, + temp_locker_enable_config: &TempLockerEnableConfig, option_connector: Option, payment_method: enums::PaymentMethod, ) -> bool { option_connector .map(|connector| { - temp_locker_disable_config + temp_locker_enable_config .0 .get(&connector) - //should be true only if payment_method is not in the disable payment_method list for connector - .map(|config| !config.payment_method.contains(&payment_method)) - .unwrap_or(true) + .map(|config| config.payment_method.contains(&payment_method)) + .unwrap_or(false) }) .unwrap_or(true) } @@ -2657,6 +2656,7 @@ pub fn router_data_type_conversion( test_mode: router_data.test_mode, connector_api_version: router_data.connector_api_version, connector_http_status_code: router_data.connector_http_status_code, + external_latency: router_data.external_latency, apple_pay_flow: router_data.apple_pay_flow, } } diff --git a/crates/router/src/core/payments/operations/payment_complete_authorize.rs b/crates/router/src/core/payments/operations/payment_complete_authorize.rs index db2c9e27c9b9..506a9b4421cd 100644 --- a/crates/router/src/core/payments/operations/payment_complete_authorize.rs +++ b/crates/router/src/core/payments/operations/payment_complete_authorize.rs @@ -115,7 +115,7 @@ impl let should_validate_pm_or_token_given = //this validation should happen if data was stored in the vault helpers::should_store_payment_method_data_in_vault( - &state.conf.temp_locker_disable_config, + &state.conf.temp_locker_enable_config, payment_attempt.connector.clone(), payment_method, ); @@ -300,11 +300,6 @@ impl Domain { let (op, payment_method_data) = helpers::make_pm_data(Box::new(self), state, payment_data).await?; - - utils::when(payment_method_data.is_none(), || { - Err(errors::ApiErrorResponse::PaymentMethodNotFound) - })?; - Ok((op, payment_method_data)) } diff --git a/crates/router/src/core/payments/transformers.rs b/crates/router/src/core/payments/transformers.rs index a8069b4d4a62..e1ea8a063592 100644 --- a/crates/router/src/core/payments/transformers.rs +++ b/crates/router/src/core/payments/transformers.rs @@ -1,13 +1,12 @@ use std::{fmt::Debug, marker::PhantomData, str::FromStr}; use api_models::payments::FrmMessage; -use common_utils::fp_utils; -use data_models::mandates::MandateData; +use common_utils::{consts::X_HS_LATENCY, fp_utils}; use diesel_models::ephemeral_key; use error_stack::{IntoReport, ResultExt}; use router_env::{instrument, tracing}; -use super::{flows::Feature, PaymentAddress, PaymentData}; +use super::{flows::Feature, PaymentData}; use crate::{ configs::settings::{ConnectorRequestReferenceIdConfig, Server}, connector::Nexinets, @@ -162,6 +161,7 @@ where payment_method_balance: None, connector_api_version, connector_http_status_code: None, + external_latency: None, apple_pay_flow, }; @@ -183,6 +183,8 @@ where operation: Op, connector_request_reference_id_config: &ConnectorRequestReferenceIdConfig, connector_http_status_code: Option, + external_latency: Option, + is_latency_header_enabled: Option, ) -> RouterResponse; } @@ -201,43 +203,39 @@ where operation: Op, connector_request_reference_id_config: &ConnectorRequestReferenceIdConfig, connector_http_status_code: Option, + external_latency: Option, + is_latency_header_enabled: Option, ) -> RouterResponse { - let captures = payment_data - .multiple_capture_data - .and_then(|multiple_capture_data| { - multiple_capture_data - .expand_captures - .and_then(|should_expand| { - should_expand.then_some( - multiple_capture_data - .get_all_captures() - .into_iter() - .cloned() - .collect(), - ) - }) - }); + let captures = + payment_data + .multiple_capture_data + .clone() + .and_then(|multiple_capture_data| { + multiple_capture_data + .expand_captures + .and_then(|should_expand| { + should_expand.then_some( + multiple_capture_data + .get_all_captures() + .into_iter() + .cloned() + .collect(), + ) + }) + }); + payments_to_payments_response( req, - payment_data.payment_attempt, - payment_data.payment_intent, - payment_data.refunds, - payment_data.disputes, - payment_data.attempts, + payment_data, captures, - payment_data.payment_method_data, customer, auth_flow, - payment_data.address, server, - payment_data.connector_response.authentication_data, &operation, - payment_data.ephemeral_key, - payment_data.sessions_token, - payment_data.frm_message, - payment_data.setup_mandate, connector_request_reference_id_config, connector_http_status_code, + external_latency, + is_latency_header_enabled, ) } } @@ -258,6 +256,8 @@ where _operation: Op, _connector_request_reference_id_config: &ConnectorRequestReferenceIdConfig, _connector_http_status_code: Option, + _external_latency: Option, + _is_latency_header_enabled: Option, ) -> RouterResponse { Ok(services::ApplicationResponse::JsonWithHeaders(( Self { @@ -290,6 +290,8 @@ where _operation: Op, _connector_request_reference_id_config: &ConnectorRequestReferenceIdConfig, _connector_http_status_code: Option, + _external_latency: Option, + _is_latency_header_enabled: Option, ) -> RouterResponse { let additional_payment_method_data: Option = data.payment_attempt @@ -333,31 +335,25 @@ where // try to use router data here so that already validated things , we don't want to repeat the validations. // Add internal value not found and external value not found so that we can give 500 / Internal server error for internal value not found #[allow(clippy::too_many_arguments)] -pub fn payments_to_payments_response( +pub fn payments_to_payments_response( payment_request: Option, - payment_attempt: storage::PaymentAttempt, - payment_intent: storage::PaymentIntent, - refunds: Vec, - disputes: Vec, - option_attempts: Option>, + payment_data: PaymentData, captures: Option>, - payment_method_data: Option, customer: Option, auth_flow: services::AuthFlow, - address: PaymentAddress, server: &Server, - redirection_data: Option, operation: &Op, - ephemeral_key_option: Option, - session_tokens: Vec, - fraud_check: Option, - mandate_data: Option, connector_request_reference_id_config: &ConnectorRequestReferenceIdConfig, connector_http_status_code: Option, + external_latency: Option, + is_latency_header_enabled: Option, ) -> RouterResponse where Op: Debug, { + let payment_attempt = payment_data.payment_attempt; + let payment_intent = payment_data.payment_intent; + let currency = payment_attempt .currency .as_ref() @@ -369,22 +365,31 @@ where field_name: "amount", })?; let mandate_id = payment_attempt.mandate_id.clone(); - let refunds_response = if refunds.is_empty() { + let refunds_response = if payment_data.refunds.is_empty() { None } else { - Some(refunds.into_iter().map(ForeignInto::foreign_into).collect()) + Some( + payment_data + .refunds + .into_iter() + .map(ForeignInto::foreign_into) + .collect(), + ) }; - let disputes_response = if disputes.is_empty() { + + let disputes_response = if payment_data.disputes.is_empty() { None } else { Some( - disputes + payment_data + .disputes .into_iter() .map(ForeignInto::foreign_into) .collect(), ) }; - let attempts_response = option_attempts.map(|attempts| { + + let attempts_response = payment_data.attempts.map(|attempts| { attempts .into_iter() .map(ForeignInto::foreign_into) @@ -419,7 +424,7 @@ where field_name: "payment_method_data", })?; let merchant_decision = payment_intent.merchant_decision.to_owned(); - let frm_message = fraud_check.map(FrmMessage::foreign_from); + let frm_message = payment_data.frm_message.map(FrmMessage::foreign_from); let payment_method_data_response = additional_payment_method_data.map(api::PaymentMethodDataResponse::from); @@ -431,23 +436,39 @@ where status_code.to_string(), )] }) - .unwrap_or(vec![]); + .unwrap_or_default(); if let Some(payment_confirm_source) = payment_intent.payment_confirm_source { headers.push(( "payment_confirm_source".to_string(), payment_confirm_source.to_string(), )) } - + if Some(true) == is_latency_header_enabled { + headers.extend( + external_latency + .map(|latency| vec![(X_HS_LATENCY.to_string(), latency.to_string())]) + .unwrap_or(vec![]), + ); + } let output = Ok(match payment_request { Some(_request) => { - if payments::is_start_pay(&operation) && redirection_data.is_some() { - let redirection_data = redirection_data.get_required_value("redirection_data")?; + if payments::is_start_pay(&operation) + && payment_data + .connector_response + .authentication_data + .is_some() + { + let redirection_data = payment_data + .connector_response + .authentication_data + .get_required_value("redirection_data")?; + let form: RedirectForm = serde_json::from_value(redirection_data) .map_err(|_| errors::ApiErrorResponse::InternalServerError)?; + services::ApplicationResponse::Form(Box::new(services::RedirectionFormData { redirect_form: form, - payment_method_data, + payment_method_data: payment_data.payment_method_data, amount, currency: currency.to_string(), })) @@ -494,22 +515,23 @@ where display_to_timestamp: wait_screen_data.display_to_timestamp, } })) - .or(redirection_data.map(|_| { - api_models::payments::NextActionData::RedirectToUrl { + .or(payment_data + .connector_response + .authentication_data + .map(|_| api_models::payments::NextActionData::RedirectToUrl { redirect_to_url: helpers::create_startpay_url( server, &payment_attempt, &payment_intent, ), - } - })); + })); }; // next action check for third party sdk session (for ex: Apple pay through trustpay has third party sdk session response) if third_party_sdk_session_next_action(&payment_attempt, operation) { next_action_response = Some( api_models::payments::NextActionData::ThirdPartySdkSessionToken { - session_token: session_tokens.get(0).cloned(), + session_token: payment_data.sessions_token.get(0).cloned(), }, ) } @@ -555,7 +577,7 @@ where ) .set_mandate_id(mandate_id) .set_mandate_data( - mandate_data.map(|d| api::MandateData { + payment_data.setup_mandate.map(|d| api::MandateData { customer_acceptance: d.customer_acceptance.map(|d| { api::CustomerAcceptance { acceptance_type: match d.acceptance_type { @@ -621,8 +643,8 @@ where .or(payment_attempt.error_message), ) .set_error_code(payment_attempt.error_code) - .set_shipping(address.shipping) - .set_billing(address.billing) + .set_shipping(payment_data.address.shipping) + .set_billing(payment_data.address.billing) .set_next_action(next_action_response) .set_return_url(payment_intent.return_url) .set_cancellation_reason(payment_attempt.cancellation_reason) @@ -642,7 +664,9 @@ where .set_allowed_payment_method_types( payment_intent.allowed_payment_method_types, ) - .set_ephemeral_key(ephemeral_key_option.map(ForeignFrom::foreign_from)) + .set_ephemeral_key( + payment_data.ephemeral_key.map(ForeignFrom::foreign_from), + ) .set_frm_message(frm_message) .set_merchant_decision(merchant_decision) .set_manual_retry_allowed(helpers::is_manual_retry_allowed( @@ -696,8 +720,8 @@ where .as_ref() .and_then(|cus| cus.phone.as_ref().map(|s| s.to_owned())), mandate_id, - shipping: address.shipping, - billing: address.billing, + shipping: payment_data.address.shipping, + billing: payment_data.address.billing, cancellation_reason: payment_attempt.cancellation_reason, payment_token: payment_attempt.payment_token, metadata: payment_intent.metadata, diff --git a/crates/router/src/core/payouts.rs b/crates/router/src/core/payouts.rs index fcb7fc571d51..bf438a15a45b 100644 --- a/crates/router/src/core/payouts.rs +++ b/crates/router/src/core/payouts.rs @@ -1205,10 +1205,7 @@ pub async fn payout_create_db_entries( .set_recurring(req.recurring.unwrap_or(false)) .set_auto_fulfill(req.auto_fulfill.unwrap_or(false)) .set_return_url(req.return_url.to_owned()) - .set_entity_type( - req.entity_type - .unwrap_or(api_enums::PayoutEntityType::default()), - ) + .set_entity_type(req.entity_type.unwrap_or_default()) .set_metadata(req.metadata.to_owned()) .set_created_at(Some(common_utils::date_time::now())) .set_last_modified_at(Some(common_utils::date_time::now())) diff --git a/crates/router/src/core/refunds.rs b/crates/router/src/core/refunds.rs index 4ad8344501c4..4f1537d5f483 100644 --- a/crates/router/src/core/refunds.rs +++ b/crates/router/src/core/refunds.rs @@ -861,13 +861,13 @@ pub async fn sync_refund_with_gateway_workflow( .await? } _ => { - payment_sync::retry_sync_task( + _ = payment_sync::retry_sync_task( &*state.store, response.connector, response.merchant_id, refund_tracker.to_owned(), ) - .await? + .await?; } } diff --git a/crates/router/src/core/utils.rs b/crates/router/src/core/utils.rs index 4400c5d5f431..908bd1438762 100644 --- a/crates/router/src/core/utils.rs +++ b/crates/router/src/core/utils.rs @@ -187,6 +187,7 @@ pub async fn construct_payout_router_data<'a, F>( payment_method_balance: None, connector_api_version: None, connector_http_status_code: None, + external_latency: None, apple_pay_flow: None, }; @@ -326,6 +327,7 @@ pub async fn construct_refund_router_data<'a, F>( payment_method_balance: None, connector_api_version, connector_http_status_code: None, + external_latency: None, apple_pay_flow: None, }; @@ -553,6 +555,7 @@ pub async fn construct_accept_dispute_router_data<'a>( payment_method_balance: None, connector_api_version: None, connector_http_status_code: None, + external_latency: None, apple_pay_flow: None, }; Ok(router_data) @@ -638,6 +641,7 @@ pub async fn construct_submit_evidence_router_data<'a>( test_mode, connector_api_version: None, connector_http_status_code: None, + external_latency: None, apple_pay_flow: None, }; Ok(router_data) @@ -728,6 +732,7 @@ pub async fn construct_upload_file_router_data<'a>( test_mode, connector_api_version: None, connector_http_status_code: None, + external_latency: None, apple_pay_flow: None, }; Ok(router_data) @@ -816,6 +821,7 @@ pub async fn construct_defend_dispute_router_data<'a>( test_mode, connector_api_version: None, connector_http_status_code: None, + external_latency: None, apple_pay_flow: None, }; Ok(router_data) @@ -897,6 +903,7 @@ pub async fn construct_retrieve_file_router_data<'a>( test_mode, connector_api_version: None, connector_http_status_code: None, + external_latency: None, apple_pay_flow: None, }; Ok(router_data) diff --git a/crates/router/src/core/webhooks.rs b/crates/router/src/core/webhooks.rs index 530d445b50de..7540845c343f 100644 --- a/crates/router/src/core/webhooks.rs +++ b/crates/router/src/core/webhooks.rs @@ -695,11 +695,6 @@ pub async fn create_event_and_trigger_outgoing_webhook(merchant_account, outgoing_webhook, &state).await; diff --git a/crates/router/src/core/webhooks/utils.rs b/crates/router/src/core/webhooks/utils.rs index 05aafd872e3a..ac7e5081c823 100644 --- a/crates/router/src/core/webhooks/utils.rs +++ b/crates/router/src/core/webhooks/utils.rs @@ -118,6 +118,7 @@ pub async fn construct_webhook_router_data<'a>( payment_method_balance: None, connector_api_version: None, connector_http_status_code: None, + external_latency: None, apple_pay_flow: None, }; Ok(router_data) diff --git a/crates/router/src/db.rs b/crates/router/src/db.rs index 389229742f33..356c1c6a512f 100644 --- a/crates/router/src/db.rs +++ b/crates/router/src/db.rs @@ -23,13 +23,17 @@ pub mod payouts; pub mod refund; pub mod reverse_lookup; +use std::fmt::Debug; + use data_models::payments::{ payment_attempt::PaymentAttemptInterface, payment_intent::PaymentIntentInterface, }; use masking::PeekInterface; +use redis_interface::errors::RedisError; +use serde::de; use storage_impl::{redis::kv_store::RedisConnInterface, MockDb}; -use crate::services::Store; +use crate::{consts, errors::CustomResult, services::Store}; #[derive(PartialEq, Eq)] pub enum StorageImpl { @@ -115,7 +119,7 @@ pub async fn get_and_deserialize_key( db: &dyn StorageInterface, key: &str, type_name: &'static str, -) -> common_utils::errors::CustomResult +) -> CustomResult where T: serde::de::DeserializeOwned, { @@ -128,4 +132,68 @@ where .change_context(redis_interface::errors::RedisError::JsonDeserializationFailed) } +pub enum KvOperation<'a, S: serde::Serialize + Debug> { + Hset((&'a str, String)), + SetNx(S), + HSetNx(&'a str, S), + Get(&'a str), + Scan(&'a str), +} + +#[derive(router_derive::TryGetEnumVariant)] +#[error(RedisError(UnknownResult))] +pub enum KvResult { + Get(T), + Hset(()), + SetNx(redis_interface::SetnxReply), + HSetNx(redis_interface::HsetnxReply), + Scan(Vec), +} + +pub async fn kv_wrapper<'a, T, S>( + store: &Store, + op: KvOperation<'a, S>, + key: impl AsRef, +) -> CustomResult, RedisError> +where + T: de::DeserializeOwned, + S: serde::Serialize + Debug, +{ + let redis_conn = store.get_redis_conn()?; + + let key = key.as_ref(); + let type_name = std::any::type_name::(); + + match op { + KvOperation::Hset(value) => { + redis_conn + .set_hash_fields(key, value, Some(consts::KV_TTL)) + .await?; + Ok(KvResult::Hset(())) + } + KvOperation::Get(field) => { + let result = redis_conn + .get_hash_field_and_deserialize(key, field, type_name) + .await?; + Ok(KvResult::Get(result)) + } + KvOperation::Scan(pattern) => { + let result: Vec = redis_conn.hscan_and_deserialize(key, pattern, None).await?; + Ok(KvResult::Scan(result)) + } + KvOperation::HSetNx(field, value) => { + let result = redis_conn + .serialize_and_set_hash_field_if_not_exist(key, field, value, Some(consts::KV_TTL)) + .await?; + Ok(KvResult::HSetNx(result)) + } + KvOperation::SetNx(value) => { + let result = redis_conn + .serialize_and_set_key_if_not_exist(key, value, Some(consts::KV_TTL.into())) + .await?; + Ok(KvResult::SetNx(result)) + } + } +} + dyn_clone::clone_trait_object!(StorageInterface); diff --git a/crates/router/src/db/address.rs b/crates/router/src/db/address.rs index 904f91613359..dda6838c9739 100644 --- a/crates/router/src/db/address.rs +++ b/crates/router/src/db/address.rs @@ -244,7 +244,7 @@ mod storage { use error_stack::{IntoReport, ResultExt}; use redis_interface::HsetnxReply; use router_env::{instrument, tracing}; - use storage_impl::redis::kv_store::{PartitionKey, RedisConnInterface}; + use storage_impl::redis::kv_store::{kv_wrapper, KvOperation, PartitionKey}; use super::AddressInterface; use crate::{ @@ -304,12 +304,18 @@ mod storage { let address = match storage_scheme { MerchantStorageScheme::PostgresOnly => database_call().await, MerchantStorageScheme::RedisKv => { - let key = format!("{}_{}", merchant_id, payment_id); + let key = format!("mid_{}_pid_{}", merchant_id, payment_id); let field = format!("add_{}", address_id); db_utils::try_redis_get_else_try_database_get( - self.get_redis_conn() - .change_context(errors::StorageError::DatabaseConnectionError)? - .get_hash_field_and_deserialize(&key, &field, "Address"), + async { + kv_wrapper( + self, + KvOperation::::HGet(&field), + key, + ) + .await? + .try_into_hget() + }, database_call, ) .await @@ -372,7 +378,7 @@ mod storage { .await } MerchantStorageScheme::RedisKv => { - let key = format!("{}_{}", merchant_id, payment_id); + let key = format!("mid_{}_pid_{}", merchant_id, payment_id); let field = format!("add_{}", &address_new.address_id); let created_address = diesel_models::Address { id: Some(0i32), @@ -394,11 +400,15 @@ mod storage { merchant_id: address_new.merchant_id.clone(), payment_id: address_new.payment_id.clone(), }; - match self - .get_redis_conn() - .map_err(Into::::into)? - .serialize_and_set_hash_field_if_not_exist(&key, &field, &created_address) - .await + + match kv_wrapper::( + self, + KvOperation::HSetNx(&field, &created_address), + &key, + ) + .await + .change_context(errors::StorageError::KVError)? + .try_into_hsetnx() { Ok(HsetnxReply::KeyNotSet) => Err(errors::StorageError::DuplicateValue { entity: "address", diff --git a/crates/router/src/db/connector_response.rs b/crates/router/src/db/connector_response.rs index 331aea03d469..c00eccb4d9e6 100644 --- a/crates/router/src/db/connector_response.rs +++ b/crates/router/src/db/connector_response.rs @@ -100,7 +100,7 @@ mod storage { use error_stack::{IntoReport, ResultExt}; use redis_interface::HsetnxReply; use router_env::{instrument, tracing}; - use storage_impl::redis::kv_store::{PartitionKey, RedisConnInterface}; + use storage_impl::redis::kv_store::{kv_wrapper, KvOperation, PartitionKey}; use super::Store; use crate::{ @@ -131,7 +131,7 @@ mod storage { let payment_id = &connector_response.payment_id; let attempt_id = &connector_response.attempt_id; - let key = format!("{merchant_id}_{payment_id}"); + let key = format!("mid_{merchant_id}_pid_{payment_id}"); let field = format!("connector_resp_{merchant_id}_{payment_id}_{attempt_id}"); let created_connector_resp = storage_type::ConnectorResponse { @@ -148,15 +148,15 @@ mod storage { authentication_data: connector_response.authentication_data.clone(), encoded_data: connector_response.encoded_data.clone(), }; - match self - .get_redis_conn() - .map_err(|er| error_stack::report!(errors::StorageError::RedisError(er)))? - .serialize_and_set_hash_field_if_not_exist( - &key, - &field, - &created_connector_resp, - ) - .await + + match kv_wrapper::( + self, + KvOperation::HSetNx(&field, &created_connector_resp), + &key, + ) + .await + .change_context(errors::StorageError::KVError)? + .try_into_hsetnx() { Ok(HsetnxReply::KeyNotSet) => Err(errors::StorageError::DuplicateValue { entity: "address", @@ -211,19 +211,22 @@ mod storage { match storage_scheme { data_models::MerchantStorageScheme::PostgresOnly => database_call().await, data_models::MerchantStorageScheme::RedisKv => { - let key = format!("{merchant_id}_{payment_id}"); + let key = format!("mid_{merchant_id}_pid_{payment_id}"); let field = format!("connector_resp_{merchant_id}_{payment_id}_{attempt_id}"); - let redis_conn = self - .get_redis_conn() - .map_err(|er| error_stack::report!(errors::StorageError::RedisError(er)))?; - let redis_fut = redis_conn.get_hash_field_and_deserialize( - &key, - &field, - "ConnectorResponse", - ); - - db_utils::try_redis_get_else_try_database_get(redis_fut, database_call).await + db_utils::try_redis_get_else_try_database_get( + async { + kv_wrapper( + self, + KvOperation::::HGet(&field), + key, + ) + .await? + .try_into_hget() + }, + database_call, + ) + .await } } } @@ -242,7 +245,7 @@ mod storage { .map_err(Into::into) .into_report(), data_models::MerchantStorageScheme::RedisKv => { - let key = format!("{}_{}", this.merchant_id, this.payment_id); + let key = format!("mid_{}_pid_{}", this.merchant_id, this.payment_id); let updated_connector_response = connector_response_update .clone() .apply_changeset(this.clone()); @@ -255,13 +258,16 @@ mod storage { &updated_connector_response.payment_id, &updated_connector_response.attempt_id ); - let updated_connector_response = self - .get_redis_conn() - .map_err(|er| error_stack::report!(errors::StorageError::RedisError(er)))? - .set_hash_fields(&key, (&field, &redis_value)) - .await - .map(|_| updated_connector_response) - .change_context(errors::StorageError::KVError)?; + + kv_wrapper::<(), _, _>( + self, + KvOperation::Hset::((&field, redis_value)), + &key, + ) + .await + .change_context(errors::StorageError::KVError)? + .try_into_hset() + .change_context(errors::StorageError::KVError)?; let redis_entry = kv::TypedSql { op: kv::DBOperation::Update { diff --git a/crates/router/src/db/ephemeral_key.rs b/crates/router/src/db/ephemeral_key.rs index 9fc3bc26fc1e..9b7936ef2afc 100644 --- a/crates/router/src/db/ephemeral_key.rs +++ b/crates/router/src/db/ephemeral_key.rs @@ -64,6 +64,7 @@ mod storage { .serialize_and_set_multiple_hash_field_if_not_exist( &[(&secret_key, &created_ek), (&id_key, &created_ek)], "ephkey", + None, ) .await { diff --git a/crates/router/src/db/refund.rs b/crates/router/src/db/refund.rs index ccbd6a5b39df..dbf4941e08ba 100644 --- a/crates/router/src/db/refund.rs +++ b/crates/router/src/db/refund.rs @@ -270,7 +270,7 @@ mod storage { use common_utils::date_time; use error_stack::{IntoReport, ResultExt}; use redis_interface::HsetnxReply; - use storage_impl::redis::kv_store::RedisConnInterface; + use storage_impl::redis::kv_store::{kv_wrapper, KvOperation}; use super::RefundInterface; use crate::{ @@ -305,13 +305,21 @@ mod storage { enums::MerchantStorageScheme::PostgresOnly => database_call().await, enums::MerchantStorageScheme::RedisKv => { let lookup_id = format!("{merchant_id}_{internal_reference_id}"); - let lookup = self.get_lookup_by_lookup_id(&lookup_id).await?; + let lookup = self + .get_lookup_by_lookup_id(&lookup_id, storage_scheme) + .await?; let key = &lookup.pk_id; db_utils::try_redis_get_else_try_database_get( - self.get_redis_conn() - .map_err(Into::::into)? - .get_hash_field_and_deserialize(key, &lookup.sk_id, "Refund"), + async { + kv_wrapper( + self, + KvOperation::::HGet(&lookup.sk_id), + key, + ) + .await? + .try_into_hget() + }, database_call, ) .await @@ -330,7 +338,7 @@ mod storage { new.insert(&conn).await.map_err(Into::into).into_report() } enums::MerchantStorageScheme::RedisKv => { - let key = format!("{}_{}", new.merchant_id, new.payment_id); + let key = format!("mid_{}_pid_{}", new.merchant_id, new.payment_id); // TODO: need to add an application generated payment attempt id to distinguish between multiple attempts for the same payment id // Check for database presence as well Maybe use a read replica here ? let created_refund = storage_types::Refund { @@ -365,11 +373,14 @@ mod storage { "pa_{}_ref_{}", &created_refund.attempt_id, &created_refund.refund_id ); - match self - .get_redis_conn() - .map_err(Into::::into)? - .serialize_and_set_hash_field_if_not_exist(&key, &field, &created_refund) - .await + match kv_wrapper::( + self, + KvOperation::HSetNx(&field, &created_refund), + &key, + ) + .await + .change_context(errors::StorageError::KVError)? + .try_into_hsetnx() { Ok(HsetnxReply::KeyNotSet) => Err(errors::StorageError::DuplicateValue { entity: "refund", @@ -377,8 +388,6 @@ mod storage { }) .into_report(), Ok(HsetnxReply::KeySet) => { - let conn = connection::pg_connection_write(self).await?; - let mut reverse_lookups = vec![ storage_types::ReverseLookupNew { sk_id: field.clone(), @@ -416,9 +425,11 @@ mod storage { source: "refund".to_string(), }) }; - storage_types::ReverseLookupNew::batch_insert(reverse_lookups, &conn) - .await - .change_context(errors::StorageError::KVError)?; + let rev_look = reverse_lookups + .into_iter() + .map(|rev| self.insert_reverse_lookup(rev, storage_scheme)); + + futures::future::try_join_all(rev_look).await?; let redis_entry = kv::TypedSql { op: kv::DBOperation::Insert { @@ -449,21 +460,25 @@ mod storage { connector_transaction_id: &str, storage_scheme: enums::MerchantStorageScheme, ) -> CustomResult, errors::StorageError> { + let database_call = || async { + let conn = connection::pg_connection_read(self).await?; + storage_types::Refund::find_by_merchant_id_connector_transaction_id( + &conn, + merchant_id, + connector_transaction_id, + ) + .await + .map_err(Into::into) + .into_report() + }; match storage_scheme { - enums::MerchantStorageScheme::PostgresOnly => { - let conn = connection::pg_connection_read(self).await?; - storage_types::Refund::find_by_merchant_id_connector_transaction_id( - &conn, - merchant_id, - connector_transaction_id, - ) - .await - .map_err(Into::into) - .into_report() - } + enums::MerchantStorageScheme::PostgresOnly => database_call().await, enums::MerchantStorageScheme::RedisKv => { let lookup_id = format!("{merchant_id}_{connector_transaction_id}"); - let lookup = match self.get_lookup_by_lookup_id(&lookup_id).await { + let lookup = match self + .get_lookup_by_lookup_id(&lookup_id, storage_scheme) + .await + { Ok(l) => l, Err(err) => { logger::error!(?err); @@ -474,11 +489,19 @@ mod storage { let pattern = db_utils::generate_hscan_pattern_for_refund(&lookup.sk_id); - self.get_redis_conn() - .map_err(Into::::into)? - .hscan_and_deserialize(key, &pattern, None) - .await - .change_context(errors::StorageError::KVError) + db_utils::try_redis_get_else_try_database_get( + async { + kv_wrapper( + self, + KvOperation::::Scan(&pattern), + key, + ) + .await? + .try_into_scan() + }, + database_call, + ) + .await } } } @@ -498,7 +521,7 @@ mod storage { .into_report() } enums::MerchantStorageScheme::RedisKv => { - let key = format!("{}_{}", this.merchant_id, this.payment_id); + let key = format!("mid_{}_pid_{}", this.merchant_id, this.payment_id); let field = format!("pa_{}_ref_{}", &this.attempt_id, &this.refund_id); let updated_refund = refund.clone().apply_changeset(this.clone()); @@ -508,11 +531,15 @@ mod storage { ) .change_context(errors::StorageError::SerializationFailed)?; - self.get_redis_conn() - .map_err(Into::::into)? - .set_hash_fields(&key, (field, redis_value)) - .await - .change_context(errors::StorageError::KVError)?; + kv_wrapper::<(), _, _>( + self, + KvOperation::Hset::((&field, redis_value)), + &key, + ) + .await + .change_context(errors::StorageError::KVError)? + .try_into_hset() + .change_context(errors::StorageError::KVError)?; let redis_entry = kv::TypedSql { op: kv::DBOperation::Update { @@ -553,13 +580,21 @@ mod storage { enums::MerchantStorageScheme::PostgresOnly => database_call().await, enums::MerchantStorageScheme::RedisKv => { let lookup_id = format!("{merchant_id}_{refund_id}"); - let lookup = self.get_lookup_by_lookup_id(&lookup_id).await?; + let lookup = self + .get_lookup_by_lookup_id(&lookup_id, storage_scheme) + .await?; let key = &lookup.pk_id; db_utils::try_redis_get_else_try_database_get( - self.get_redis_conn() - .map_err(Into::::into)? - .get_hash_field_and_deserialize(key, &lookup.sk_id, "Refund"), + async { + kv_wrapper( + self, + KvOperation::::HGet(&lookup.sk_id), + key, + ) + .await? + .try_into_hget() + }, database_call, ) .await @@ -590,13 +625,21 @@ mod storage { enums::MerchantStorageScheme::PostgresOnly => database_call().await, enums::MerchantStorageScheme::RedisKv => { let lookup_id = format!("{merchant_id}_{connector_refund_id}_{connector}"); - let lookup = self.get_lookup_by_lookup_id(&lookup_id).await?; + let lookup = self + .get_lookup_by_lookup_id(&lookup_id, storage_scheme) + .await?; let key = &lookup.pk_id; db_utils::try_redis_get_else_try_database_get( - self.get_redis_conn() - .map_err(Into::::into)? - .get_hash_field_and_deserialize(key, &lookup.sk_id, "Refund"), + async { + kv_wrapper( + self, + KvOperation::::HGet(&lookup.sk_id), + key, + ) + .await? + .try_into_hget() + }, database_call, ) .await @@ -610,25 +653,34 @@ mod storage { merchant_id: &str, storage_scheme: enums::MerchantStorageScheme, ) -> CustomResult, errors::StorageError> { + let database_call = || async { + let conn = connection::pg_connection_read(self).await?; + storage_types::Refund::find_by_payment_id_merchant_id( + &conn, + payment_id, + merchant_id, + ) + .await + .map_err(Into::into) + .into_report() + }; match storage_scheme { - enums::MerchantStorageScheme::PostgresOnly => { - let conn = connection::pg_connection_read(self).await?; - storage_types::Refund::find_by_payment_id_merchant_id( - &conn, - payment_id, - merchant_id, + enums::MerchantStorageScheme::PostgresOnly => database_call().await, + enums::MerchantStorageScheme::RedisKv => { + let key = format!("mid_{merchant_id}_pid_{payment_id}"); + db_utils::try_redis_get_else_try_database_get( + async { + kv_wrapper( + self, + KvOperation::::Scan("pa_*_ref_*"), + key, + ) + .await? + .try_into_scan() + }, + database_call, ) .await - .map_err(Into::into) - .into_report() - } - enums::MerchantStorageScheme::RedisKv => { - let key = format!("{merchant_id}_{payment_id}"); - self.get_redis_conn() - .map_err(Into::::into)? - .hscan_and_deserialize(&key, "pa_*_ref_*", None) - .await - .change_context(errors::StorageError::KVError) } } } @@ -638,21 +690,21 @@ mod storage { &self, merchant_id: &str, refund_details: &api_models::refunds::RefundListRequest, - storage_scheme: enums::MerchantStorageScheme, + _storage_scheme: enums::MerchantStorageScheme, limit: i64, offset: i64, ) -> CustomResult, errors::StorageError> { - match storage_scheme { - enums::MerchantStorageScheme::PostgresOnly => { - let conn = connection::pg_connection_read(self).await?; - ::filter_by_constraints(&conn, merchant_id, refund_details, limit, offset) - .await - .map_err(Into::into) - .into_report() - } - - enums::MerchantStorageScheme::RedisKv => Err(errors::StorageError::KVError.into()), - } + let conn = connection::pg_connection_read(self).await?; + ::filter_by_constraints( + &conn, + merchant_id, + refund_details, + limit, + offset, + ) + .await + .map_err(Into::into) + .into_report() } #[cfg(feature = "olap")] @@ -660,19 +712,13 @@ mod storage { &self, merchant_id: &str, refund_details: &api_models::refunds::TimeRange, - storage_scheme: enums::MerchantStorageScheme, + _storage_scheme: enums::MerchantStorageScheme, ) -> CustomResult { - match storage_scheme { - enums::MerchantStorageScheme::PostgresOnly => { - let conn = connection::pg_connection_read(self).await?; - ::filter_by_meta_constraints(&conn, merchant_id, refund_details) + let conn = connection::pg_connection_read(self).await?; + ::filter_by_meta_constraints(&conn, merchant_id, refund_details) .await .map_err(Into::into) .into_report() - } - - enums::MerchantStorageScheme::RedisKv => Err(errors::StorageError::KVError.into()), - } } #[cfg(feature = "olap")] @@ -680,19 +726,17 @@ mod storage { &self, merchant_id: &str, refund_details: &api_models::refunds::RefundListRequest, - storage_scheme: enums::MerchantStorageScheme, + _storage_scheme: enums::MerchantStorageScheme, ) -> CustomResult { - match storage_scheme { - enums::MerchantStorageScheme::PostgresOnly => { - let conn = connection::pg_connection_read(self).await?; - ::get_refunds_count(&conn, merchant_id, refund_details) - .await - .map_err(Into::into) - .into_report() - } - - enums::MerchantStorageScheme::RedisKv => Err(errors::StorageError::KVError.into()), - } + let conn = connection::pg_connection_read(self).await?; + ::get_refunds_count( + &conn, + merchant_id, + refund_details, + ) + .await + .map_err(Into::into) + .into_report() } } } diff --git a/crates/router/src/db/reverse_lookup.rs b/crates/router/src/db/reverse_lookup.rs index 46e89f62e743..7e660224d5c9 100644 --- a/crates/router/src/db/reverse_lookup.rs +++ b/crates/router/src/db/reverse_lookup.rs @@ -1,10 +1,10 @@ -use error_stack::IntoReport; - -use super::{cache, MockDb, Store}; +use super::{MockDb, Store}; use crate::{ - connection, errors::{self, CustomResult}, - types::storage::reverse_lookup::{ReverseLookup, ReverseLookupNew}, + types::storage::{ + enums, + reverse_lookup::{ReverseLookup, ReverseLookupNew}, + }, }; #[async_trait::async_trait] @@ -12,35 +12,156 @@ pub trait ReverseLookupInterface { async fn insert_reverse_lookup( &self, _new: ReverseLookupNew, + _storage_scheme: enums::MerchantStorageScheme, ) -> CustomResult; async fn get_lookup_by_lookup_id( &self, _id: &str, + _storage_scheme: enums::MerchantStorageScheme, ) -> CustomResult; } -#[async_trait::async_trait] -impl ReverseLookupInterface for Store { - async fn insert_reverse_lookup( - &self, - new: ReverseLookupNew, - ) -> CustomResult { - let conn = connection::pg_connection_write(self).await?; - new.insert(&conn).await.map_err(Into::into).into_report() - } +#[cfg(not(feature = "kv_store"))] +mod storage { + use error_stack::IntoReport; - async fn get_lookup_by_lookup_id( - &self, - id: &str, - ) -> CustomResult { - let database_call = || async { + use super::{ReverseLookupInterface, Store}; + use crate::{ + connection, + errors::{self, CustomResult}, + types::storage::{ + enums, + reverse_lookup::{ReverseLookup, ReverseLookupNew}, + }, + }; + + #[async_trait::async_trait] + impl ReverseLookupInterface for Store { + async fn insert_reverse_lookup( + &self, + new: ReverseLookupNew, + _storage_scheme: enums::MerchantStorageScheme, + ) -> CustomResult { + let conn = connection::pg_connection_write(self).await?; + new.insert(&conn).await.map_err(Into::into).into_report() + } + + async fn get_lookup_by_lookup_id( + &self, + id: &str, + _storage_scheme: enums::MerchantStorageScheme, + ) -> CustomResult { let conn = connection::pg_connection_read(self).await?; ReverseLookup::find_by_lookup_id(id, &conn) .await .map_err(Into::into) .into_report() - }; - cache::get_or_populate_redis(self, format!("reverse_lookup_{id}"), database_call).await + } + } +} + +#[cfg(feature = "kv_store")] +mod storage { + use error_stack::{IntoReport, ResultExt}; + use redis_interface::SetnxReply; + use storage_impl::redis::kv_store::{kv_wrapper, KvOperation}; + + use super::{ReverseLookupInterface, Store}; + use crate::{ + connection, + errors::{self, CustomResult}, + types::storage::{ + enums, kv, + reverse_lookup::{ReverseLookup, ReverseLookupNew}, + }, + utils::{db_utils, storage_partitioning::PartitionKey}, + }; + + #[async_trait::async_trait] + impl ReverseLookupInterface for Store { + async fn insert_reverse_lookup( + &self, + new: ReverseLookupNew, + storage_scheme: enums::MerchantStorageScheme, + ) -> CustomResult { + match storage_scheme { + data_models::MerchantStorageScheme::PostgresOnly => { + let conn = connection::pg_connection_write(self).await?; + new.insert(&conn).await.map_err(Into::into).into_report() + } + data_models::MerchantStorageScheme::RedisKv => { + let created_rev_lookup = ReverseLookup { + lookup_id: new.lookup_id.clone(), + sk_id: new.sk_id.clone(), + pk_id: new.pk_id.clone(), + source: new.source.clone(), + }; + let combination = &created_rev_lookup.pk_id; + match kv_wrapper::( + self, + KvOperation::SetNx(&created_rev_lookup), + format!("reverse_lookup_{}", &created_rev_lookup.lookup_id), + ) + .await + .change_context(errors::StorageError::KVError)? + .try_into_setnx() + { + Ok(SetnxReply::KeySet) => { + let redis_entry = kv::TypedSql { + op: kv::DBOperation::Insert { + insertable: kv::Insertable::ReverseLookUp(new), + }, + }; + self.push_to_drainer_stream::( + redis_entry, + PartitionKey::MerchantIdPaymentIdCombination { combination }, + ) + .await + .change_context(errors::StorageError::KVError)?; + + Ok(created_rev_lookup) + } + Ok(SetnxReply::KeyNotSet) => Err(errors::StorageError::DuplicateValue { + entity: "reverse_lookup", + key: Some(created_rev_lookup.lookup_id.clone()), + }) + .into_report(), + Err(er) => Err(er).change_context(errors::StorageError::KVError), + } + } + } + } + + async fn get_lookup_by_lookup_id( + &self, + id: &str, + storage_scheme: enums::MerchantStorageScheme, + ) -> CustomResult { + let database_call = || async { + let conn = connection::pg_connection_read(self).await?; + ReverseLookup::find_by_lookup_id(id, &conn) + .await + .map_err(Into::into) + .into_report() + }; + + match storage_scheme { + data_models::MerchantStorageScheme::PostgresOnly => database_call().await, + data_models::MerchantStorageScheme::RedisKv => { + let redis_fut = async { + kv_wrapper( + self, + KvOperation::::Get, + format!("reverse_lookup_{id}"), + ) + .await? + .try_into_get() + }; + + db_utils::try_redis_get_else_try_database_get(redis_fut, database_call).await + } + } + } } } @@ -49,6 +170,7 @@ impl ReverseLookupInterface for MockDb { async fn insert_reverse_lookup( &self, new: ReverseLookupNew, + _storage_scheme: enums::MerchantStorageScheme, ) -> CustomResult { let reverse_lookup_insert = ReverseLookup::from(new); self.reverse_lookups @@ -57,9 +179,11 @@ impl ReverseLookupInterface for MockDb { .push(reverse_lookup_insert.clone()); Ok(reverse_lookup_insert) } + async fn get_lookup_by_lookup_id( &self, lookup_id: &str, + _storage_scheme: enums::MerchantStorageScheme, ) -> CustomResult { self.reverse_lookups .lock() diff --git a/crates/router/src/services/api.rs b/crates/router/src/services/api.rs index ec358e357f2b..def00bb3f2f4 100644 --- a/crates/router/src/services/api.rs +++ b/crates/router/src/services/api.rs @@ -13,8 +13,8 @@ use std::{ use actix_web::{body, web, FromRequest, HttpRequest, HttpResponse, Responder, ResponseError}; use api_models::enums::CaptureMethod; pub use client::{proxy_bypass_urls, ApiClient, MockApiClient, ProxyClient}; -use common_utils::errors::ReportSwitchExt; pub use common_utils::request::{ContentType, Method, Request, RequestBuilder}; +use common_utils::{consts::X_HS_LATENCY, errors::ReportSwitchExt}; use error_stack::{report, IntoReport, Report, ResultExt}; use masking::{ExposeOptionInterface, PeekInterface}; use router_env::{instrument, tracing, tracing_actix_web::RequestId, Tag}; @@ -338,7 +338,9 @@ where match connector_request { Some(request) => { logger::debug!(connector_request=?request); + let current_time = Instant::now(); let response = call_connector_api(state, request).await; + let external_latency = current_time.elapsed().as_millis(); logger::debug!(connector_response=?response); match response { Ok(body) => { @@ -363,10 +365,20 @@ where error })?; data.connector_http_status_code = connector_http_status_code; + // Add up multiple external latencies in case of multiple external calls within the same request. + data.external_latency = Some( + data.external_latency + .map_or(external_latency, |val| val + external_latency), + ); data } Err(body) => { router_data.connector_http_status_code = Some(body.status_code); + router_data.external_latency = Some( + router_data + .external_latency + .map_or(external_latency, |val| val + external_latency), + ); metrics::CONNECTOR_ERROR_RESPONSE_COUNT.add( &metrics::CONTEXT, 1, @@ -399,6 +411,11 @@ where }; router_data.response = Err(error_response); router_data.connector_http_status_code = Some(504); + router_data.external_latency = Some( + router_data + .external_latency + .map_or(external_latency, |val| val + external_latency), + ); Ok(router_data) } else { Err(error.change_context( @@ -472,11 +489,9 @@ pub async fn send_request( match request.content_type { Some(ContentType::Json) => client.json(&request.payload), - Some(ContentType::FormData) => client.multipart( - request - .form_data - .unwrap_or_else(reqwest::multipart::Form::new), - ), + Some(ContentType::FormData) => { + client.multipart(request.form_data.unwrap_or_default()) + } // Currently this is not used remove this if not required // If using this then handle the serde_part @@ -561,6 +576,7 @@ async fn handle_response( logger::info!(?response); let status_code = response.status().as_u16(); let headers = Some(response.headers().to_owned()); + match status_code { 200..=202 | 302 | 204 => { logger::debug!(response=?response); @@ -872,8 +888,15 @@ where .map_into_boxed_body() } Ok(ApplicationResponse::JsonWithHeaders((response, headers))) => { + let request_elapsed_time = request.headers().get(X_HS_LATENCY).and_then(|value| { + if value == "true" { + Some(start_instant.elapsed()) + } else { + None + } + }); match serde_json::to_string(&response) { - Ok(res) => http_response_json_with_headers(res, headers), + Ok(res) => http_response_json_with_headers(res, headers, request_elapsed_time), Err(_) => http_response_err( r#"{ "error": { @@ -950,12 +973,23 @@ pub fn http_response_json(response: T) -> HttpRe pub fn http_response_json_with_headers( response: T, - headers: Vec<(String, String)>, + mut headers: Vec<(String, String)>, + request_duration: Option, ) -> HttpResponse { let mut response_builder = HttpResponse::Ok(); - for (name, value) in headers { - response_builder.append_header((name, value)); + + for (name, value) in headers.iter_mut() { + if name == X_HS_LATENCY { + if let Some(request_duration) = request_duration { + if let Ok(external_latency) = value.parse::() { + let updated_duration = request_duration.as_millis() - external_latency; + *value = updated_duration.to_string(); + } + } + } + response_builder.append_header((name.clone(), value.clone())); } + response_builder .content_type(mime::APPLICATION_JSON) .body(response) diff --git a/crates/router/src/types.rs b/crates/router/src/types.rs index 70cac98c5028..560d706a8403 100644 --- a/crates/router/src/types.rs +++ b/crates/router/src/types.rs @@ -289,7 +289,7 @@ pub struct RouterData { pub test_mode: Option, pub connector_http_status_code: Option, - + pub external_latency: Option, /// Contains apple pay flow type simplified or manual pub apple_pay_flow: Option, } @@ -1117,6 +1117,7 @@ impl From<(&RouterData, T2)> payment_method_balance: data.payment_method_balance.clone(), connector_api_version: data.connector_api_version.clone(), connector_http_status_code: data.connector_http_status_code, + external_latency: data.external_latency, apple_pay_flow: data.apple_pay_flow.clone(), } } @@ -1171,6 +1172,7 @@ impl payment_method_balance: None, connector_api_version: None, connector_http_status_code: data.connector_http_status_code, + external_latency: data.external_latency, apple_pay_flow: None, } } diff --git a/crates/router/src/types/transformers.rs b/crates/router/src/types/transformers.rs index f81cf3b6dfe6..04334fc201ff 100644 --- a/crates/router/src/types/transformers.rs +++ b/crates/router/src/types/transformers.rs @@ -2,6 +2,7 @@ use actix_web::http::header::HeaderMap; use api_models::{enums as api_enums, payments}; use common_utils::{ + consts::X_HS_LATENCY, crypto::Encryptable, ext_traits::{StringExt, ValueExt}, pii, @@ -453,7 +454,6 @@ impl ForeignTryFrom for storage_enum impl ForeignFrom for api_types::Config { fn foreign_from(config: storage::Config) -> Self { - let config = config; Self { key: config.key, value: config.config, @@ -472,7 +472,6 @@ impl<'a> ForeignFrom<&'a api_types::ConfigUpdate> for storage::ConfigUpdate { impl<'a> From<&'a domain::Address> for api_types::Address { fn from(address: &domain::Address) -> Self { - let address = address; Self { address: Some(api_types::AddressDetails { city: address.city.clone(), @@ -789,8 +788,14 @@ impl ForeignTryFrom<&HeaderMap> for api_models::payments::HeaderPayload { ) }) .transpose()?; + + let x_hs_latency = get_header_value_by_key(X_HS_LATENCY.into(), headers) + .map(|value| value == Some("true")) + .unwrap_or(false); + Ok(Self { payment_confirm_source, + x_hs_latency: Some(x_hs_latency), }) } } diff --git a/crates/router/src/utils.rs b/crates/router/src/utils.rs index e8a5597eb26d..bf2f5943fc20 100644 --- a/crates/router/src/utils.rs +++ b/crates/router/src/utils.rs @@ -5,6 +5,8 @@ pub mod ext_traits; #[cfg(feature = "kv_store")] pub mod storage_partitioning; +use std::fmt::Debug; + use api_models::{enums, payments, webhooks}; use base64::Engine; pub use common_utils::{ @@ -27,11 +29,12 @@ use crate::{ consts, core::{ errors::{self, CustomResult, RouterResult, StorageErrorExt}, - utils, + utils, webhooks as webhooks_core, }, db::StorageInterface, logger, routes::metrics, + services, types::{ self, domain::{ @@ -39,6 +42,7 @@ use crate::{ types::{encrypt_optional, AsyncLift}, }, storage, + transformers::{ForeignTryFrom, ForeignTryInto}, }, }; @@ -669,3 +673,91 @@ pub fn add_apple_pay_payment_status_metrics( } } } + +impl ForeignTryFrom for enums::EventType { + type Error = errors::ValidationError; + + fn foreign_try_from(value: enums::IntentStatus) -> Result { + match value { + enums::IntentStatus::Succeeded => Ok(Self::PaymentSucceeded), + enums::IntentStatus::Failed => Ok(Self::PaymentFailed), + enums::IntentStatus::Processing => Ok(Self::PaymentProcessing), + enums::IntentStatus::RequiresMerchantAction + | enums::IntentStatus::RequiresCustomerAction => Ok(Self::ActionRequired), + _ => Err(errors::ValidationError::IncorrectValueProvided { + field_name: "intent_status", + }), + } + } +} + +pub async fn trigger_payments_webhook( + merchant_account: domain::MerchantAccount, + payment_data: crate::core::payments::PaymentData, + req: Option, + customer: Option, + state: &crate::routes::AppState, + operation: Op, +) -> RouterResult<()> +where + F: Send + Clone + Sync, + Op: Debug, +{ + let status = payment_data.payment_intent.status; + let payment_id = payment_data.payment_intent.payment_id.clone(); + let captures = payment_data + .multiple_capture_data + .clone() + .map(|multiple_capture_data| { + multiple_capture_data + .get_all_captures() + .into_iter() + .cloned() + .collect() + }); + + if matches!( + status, + enums::IntentStatus::Succeeded | enums::IntentStatus::Failed + ) { + let payments_response = crate::core::payments::transformers::payments_to_payments_response( + req, + payment_data, + captures, + customer, + services::AuthFlow::Merchant, + &state.conf.server, + &operation, + &state.conf.connector_request_reference_id_config, + None, + None, + None, + )?; + + let event_type: enums::EventType = status + .foreign_try_into() + .into_report() + .change_context(errors::ApiErrorResponse::WebhookProcessingFailure) + .attach_printable("payment event type mapping failed")?; + + if let services::ApplicationResponse::JsonWithHeaders((payments_response_json, _)) = + payments_response + { + Box::pin( + webhooks_core::create_event_and_trigger_appropriate_outgoing_webhook( + state.clone(), + merchant_account, + event_type, + diesel_models::enums::EventClass::Payments, + None, + payment_id, + diesel_models::enums::EventObjectType::PaymentDetails, + webhooks::OutgoingWebhookContent::PaymentDetails(payments_response_json), + ), + ) + .await?; + } + } + + Ok(()) +} diff --git a/crates/router/src/workflows/payment_sync.rs b/crates/router/src/workflows/payment_sync.rs index 4dbf97081a60..4ca0c4bccd7b 100644 --- a/crates/router/src/workflows/payment_sync.rs +++ b/crates/router/src/workflows/payment_sync.rs @@ -4,11 +4,13 @@ use router_env::logger; use scheduler::{ consumer::{self, types::process_data, workflows::ProcessTrackerWorkflow}, db::process_tracker::ProcessTrackerExt, - errors as sch_errors, utils, SchedulerAppState, + errors as sch_errors, utils as scheduler_utils, SchedulerAppState, }; use crate::{ + consts, core::{ + errors::StorageErrorExt, payment_methods::Oss, payments::{self as payment_flows, operations}, }, @@ -20,6 +22,7 @@ use crate::{ api, storage::{self, enums}, }, + utils, }; pub struct PaymentsSyncWorkflow; @@ -57,7 +60,7 @@ impl ProcessTrackerWorkflow for PaymentsSyncWorkflow { ) .await?; - let (payment_data, _, _, _) = + let (mut payment_data, _, customer, _, _) = payment_flows::payments_operation_core::( state, merchant_account.clone(), @@ -93,15 +96,72 @@ impl ProcessTrackerWorkflow for PaymentsSyncWorkflow { let connector = payment_data .payment_attempt .connector + .clone() .ok_or(sch_errors::ProcessTrackerError::MissingRequiredField)?; - retry_sync_task( + let is_last_retry = retry_sync_task( db, connector, - payment_data.payment_attempt.merchant_id, + payment_data.payment_attempt.merchant_id.clone(), process, ) - .await? + .await?; + + // If the payment status is still processing and there is no connector transaction_id + // then change the payment status to failed if all retries exceeded + if is_last_retry + && payment_data.payment_attempt.status == enums::AttemptStatus::Pending + && payment_data + .payment_attempt + .connector_transaction_id + .as_ref() + .is_none() + { + let payment_intent_update = data_models::payments::payment_intent::PaymentIntentUpdate::PGStatusUpdate { status: api_models::enums::IntentStatus::Failed }; + let payment_attempt_update = + data_models::payments::payment_attempt::PaymentAttemptUpdate::ErrorUpdate { + connector: None, + status: api_models::enums::AttemptStatus::AuthenticationFailed, + error_code: None, + error_message: None, + error_reason: Some(Some( + consts::REQUEST_TIMEOUT_ERROR_MESSAGE_FROM_PSYNC.to_string(), + )), + amount_capturable: Some(0), + }; + + payment_data.payment_attempt = db + .update_payment_attempt_with_attempt_id( + payment_data.payment_attempt, + payment_attempt_update, + merchant_account.storage_scheme, + ) + .await + .to_not_found_response(errors::ApiErrorResponse::PaymentNotFound)?; + + payment_data.payment_intent = db + .update_payment_intent( + payment_data.payment_intent, + payment_intent_update, + merchant_account.storage_scheme, + ) + .await + .to_not_found_response(errors::ApiErrorResponse::PaymentNotFound)?; + + // Trigger the outgoing webhook to notify the merchant about failed payment + let operation = operations::PaymentStatus; + utils::trigger_payments_webhook::<_, api_models::payments::PaymentsRequest, _>( + merchant_account, + payment_data, + None, + customer, + state, + operation, + ) + .await + .map_err(|error| logger::warn!(payments_outgoing_webhook_error=?error)) + .ok(); + } } }; Ok(()) @@ -117,6 +177,26 @@ impl ProcessTrackerWorkflow for PaymentsSyncWorkflow { } } +/// Get the next schedule time +/// +/// The schedule time can be configured in configs by this key `pt_mapping_trustpay` +/// ```json +/// { +/// "default_mapping": { +/// "start_after": 60, +/// "frequency": [300], +/// "count": [5] +/// }, +/// "max_retries_count": 5 +/// } +/// ``` +/// +/// This config represents +/// +/// `start_after`: The first psync should happen after 60 seconds +/// +/// `frequency` and `count`: The next 5 retries should have an interval of 300 seconds between them +/// pub async fn get_sync_process_schedule_time( db: &dyn StorageInterface, connector: &str, @@ -142,25 +222,32 @@ pub async fn get_sync_process_schedule_time( process_data::ConnectorPTMapping::default() } }; - let time_delta = utils::get_schedule_time(mapping, merchant_id, retry_count + 1); + let time_delta = scheduler_utils::get_schedule_time(mapping, merchant_id, retry_count + 1); - Ok(utils::get_time_from_delta(time_delta)) + Ok(scheduler_utils::get_time_from_delta(time_delta)) } +/// Schedule the task for retry +/// +/// Returns bool which indicates whether this was the last retry or not pub async fn retry_sync_task( db: &dyn StorageInterface, connector: String, merchant_id: String, pt: storage::ProcessTracker, -) -> Result<(), sch_errors::ProcessTrackerError> { +) -> Result { let schedule_time = get_sync_process_schedule_time(db, &connector, &merchant_id, pt.retry_count).await?; match schedule_time { - Some(s_time) => pt.retry(db.as_scheduler(), s_time).await, + Some(s_time) => { + pt.retry(db.as_scheduler(), s_time).await?; + Ok(false) + } None => { pt.finish_with_status(db.as_scheduler(), "RETRIES_EXCEEDED".to_string()) - .await + .await?; + Ok(true) } } } @@ -173,9 +260,11 @@ mod tests { #[test] fn test_get_default_schedule_time() { let schedule_time_delta = - utils::get_schedule_time(process_data::ConnectorPTMapping::default(), "-", 0).unwrap(); + scheduler_utils::get_schedule_time(process_data::ConnectorPTMapping::default(), "-", 0) + .unwrap(); let first_retry_time_delta = - utils::get_schedule_time(process_data::ConnectorPTMapping::default(), "-", 1).unwrap(); + scheduler_utils::get_schedule_time(process_data::ConnectorPTMapping::default(), "-", 1) + .unwrap(); let cpt_default = process_data::ConnectorPTMapping::default().default_mapping; assert_eq!( vec![schedule_time_delta, first_retry_time_delta], diff --git a/crates/router/tests/connectors/aci.rs b/crates/router/tests/connectors/aci.rs index d56958ac80e6..63d2ee364fa6 100644 --- a/crates/router/tests/connectors/aci.rs +++ b/crates/router/tests/connectors/aci.rs @@ -92,6 +92,7 @@ fn construct_payment_router_data() -> types::PaymentsAuthorizeRouterData { connector_api_version: None, connector_http_status_code: None, apple_pay_flow: None, + external_latency: None, } } @@ -148,6 +149,7 @@ fn construct_refund_router_data() -> types::RefundsRouterData { connector_api_version: None, connector_http_status_code: None, apple_pay_flow: None, + external_latency: None, } } diff --git a/crates/router/tests/connectors/utils.rs b/crates/router/tests/connectors/utils.rs index 05690f9613c9..f890eec192f2 100644 --- a/crates/router/tests/connectors/utils.rs +++ b/crates/router/tests/connectors/utils.rs @@ -514,6 +514,7 @@ pub trait ConnectorActions: Connector { connector_api_version: None, connector_http_status_code: None, apple_pay_flow: None, + external_latency: None, } } diff --git a/crates/router_derive/src/lib.rs b/crates/router_derive/src/lib.rs index 58d623682ac5..3f34c156ae8f 100644 --- a/crates/router_derive/src/lib.rs +++ b/crates/router_derive/src/lib.rs @@ -538,3 +538,41 @@ pub fn validate_config(input: proc_macro::TokenStream) -> proc_macro::TokenStrea .unwrap_or_else(|error| error.into_compile_error()) .into() } + +/// Generates the function to get the value out of enum variant +/// Usage +/// ``` +/// #[derive(TryGetEnumVariant)] +/// #[error(RedisError(UnknownResult))] +/// enum Result { +/// Set(String), +/// Get(i32) +/// } +/// ``` +/// +/// This will generate the function to get `String` and `i32` out of the variants +/// +/// ``` +/// impl Result { +/// fn try_into_get(&self)-> Result { +/// match self { +/// Self::Get(a) => Ok(a), +/// _=>Err(RedisError::UnknownResult) +/// } +/// } +/// +/// fn try_into_set(&self)-> Result { +/// match self { +/// Self::Set(a) => Ok(a), +/// _=> Err(RedisError::UnknownResult) +/// } +/// } +/// } +#[proc_macro_derive(TryGetEnumVariant, attributes(error))] +pub fn try_get_enum_variant(input: proc_macro::TokenStream) -> proc_macro::TokenStream { + let input = syn::parse_macro_input!(input as syn::DeriveInput); + + macros::try_get_enum::try_get_enum_variant(input) + .unwrap_or_else(|error| error.into_compile_error()) + .into() +} diff --git a/crates/router_derive/src/macros.rs b/crates/router_derive/src/macros.rs index 6f5de3833045..86501f054a59 100644 --- a/crates/router_derive/src/macros.rs +++ b/crates/router_derive/src/macros.rs @@ -3,6 +3,7 @@ pub(crate) mod diesel; pub(crate) mod generate_schema; pub(crate) mod misc; pub(crate) mod operation; +pub(crate) mod try_get_enum; mod helpers; diff --git a/crates/router_derive/src/macros/try_get_enum.rs b/crates/router_derive/src/macros/try_get_enum.rs new file mode 100644 index 000000000000..3a534b080df1 --- /dev/null +++ b/crates/router_derive/src/macros/try_get_enum.rs @@ -0,0 +1,122 @@ +use proc_macro2::Span; +use syn::punctuated::Punctuated; + +/// Try and get the variants for an enum +pub fn try_get_enum_variant( + input: syn::DeriveInput, +) -> Result { + let name = &input.ident; + + let error_attr = input + .attrs + .iter() + .find(|attr| attr.path.is_ident("error")) + .ok_or(super::helpers::syn_error( + proc_macro2::Span::call_site(), + "Unable to find attribute error. Expected #[error(..)]", + ))?; + let (error_type, error_variant) = get_error_type_and_variant(error_attr)?; + let (impl_generics, generics, where_clause) = input.generics.split_for_impl(); + + let variants = get_enum_variants(&input.data)?; + + let try_into_fns = variants.iter().map(|variant| { + let variant_name = &variant.ident; + let variant_field = get_enum_variant_field(variant)?; + let variant_types = variant_field.iter().map(|f|f.ty.clone()); + + let try_into_fn = syn::Ident::new( + &format!("try_into_{}", variant_name.to_string().to_lowercase()), + proc_macro2::Span::call_site(), + ); + + Ok(quote::quote! { + pub fn #try_into_fn(self)->Result<(#(#variant_types),*),error_stack::Report<#error_type>> { + match self { + Self::#variant_name(inner) => Ok(inner), + _=> Err(error_stack::report!(#error_type::#error_variant)), + } + } + }) + }).collect::,syn::Error>>()?; + + let expanded = quote::quote! { + impl #impl_generics #name #generics #where_clause { + #(#try_into_fns)* + } + }; + + Ok(expanded) +} + +/// Parses the attribute #[error(ErrorType(ErrorVariant))] +fn get_error_type_and_variant(attr: &syn::Attribute) -> syn::Result<(syn::Ident, syn::Path)> { + let meta = attr.parse_meta()?; + let metalist = match meta { + syn::Meta::List(list) => list, + _ => { + return Err(super::helpers::syn_error( + proc_macro2::Span::call_site(), + "Invalid attribute format #[error(ErrorType(ErrorVariant)]", + )) + } + }; + + for meta in metalist.nested.iter() { + if let syn::NestedMeta::Meta(syn::Meta::List(meta)) = meta { + let error_type = meta + .path + .get_ident() + .ok_or(super::helpers::syn_error( + proc_macro2::Span::call_site(), + "Invalid attribute format #[error(ErrorType(ErrorVariant))]", + )) + .cloned()?; + let error_variant = get_error_variant(meta)?; + return Ok((error_type, error_variant)); + }; + } + + Err(super::helpers::syn_error( + proc_macro2::Span::call_site(), + "Invalid attribute format #[error(ErrorType(ErrorVariant))]", + )) +} + +fn get_error_variant(meta: &syn::MetaList) -> syn::Result { + for meta in meta.nested.iter() { + if let syn::NestedMeta::Meta(syn::Meta::Path(meta)) = meta { + return Ok(meta.clone()); + } + } + Err(super::helpers::syn_error( + proc_macro2::Span::call_site(), + "Invalid attribute format expected #[error(ErrorType(ErrorVariant))]", + )) +} + +/// Get variants from Enum +fn get_enum_variants(data: &syn::Data) -> syn::Result> { + if let syn::Data::Enum(syn::DataEnum { variants, .. }) = data { + Ok(variants.clone()) + } else { + Err(super::helpers::non_enum_error()) + } +} + +/// Get Field from an enum variant +fn get_enum_variant_field( + variant: &syn::Variant, +) -> syn::Result> { + let field = match variant.fields.clone() { + syn::Fields::Unnamed(un) => un.unnamed, + syn::Fields::Named(n) => n.named, + syn::Fields::Unit => { + return Err(super::helpers::syn_error( + Span::call_site(), + "The enum is a unit variant it's not supported", + )) + } + }; + Ok(field) +} diff --git a/crates/router_env/Cargo.toml b/crates/router_env/Cargo.toml index c72220f52d22..1181685d723b 100644 --- a/crates/router_env/Cargo.toml +++ b/crates/router_env/Cargo.toml @@ -10,6 +10,7 @@ license.workspace = true [dependencies] cargo_metadata = "0.15.4" config = { version = "0.13.3", features = ["toml"] } +error-stack = "0.3.1" gethostname = "0.4.3" once_cell = "1.18.0" opentelemetry = { version = "0.19.0", features = ["rt-tokio-current-thread", "metrics"] } diff --git a/crates/router_env/src/logger/config.rs b/crates/router_env/src/logger/config.rs index 269759ed44f6..664c0d508f52 100644 --- a/crates/router_env/src/logger/config.rs +++ b/crates/router_env/src/logger/config.rs @@ -101,6 +101,8 @@ pub struct LogTelemetry { pub otel_exporter_otlp_timeout: Option, /// Whether to use xray ID generator, (enable this if you plan to use AWS-XRAY) pub use_xray_generator: bool, + /// Route Based Tracing + pub route_to_trace: Option>, } /// Telemetry / tracing. diff --git a/crates/router_env/src/logger/setup.rs b/crates/router_env/src/logger/setup.rs index 313e64d0e9c1..992de3e747e4 100644 --- a/crates/router_env/src/logger/setup.rs +++ b/crates/router_env/src/logger/setup.rs @@ -12,6 +12,7 @@ use opentelemetry::{ trace::BatchConfig, Resource, }, + trace::{TraceContextExt, TraceState}, KeyValue, }; use opentelemetry_otlp::{TonicExporterBuilder, WithExportConfig}; @@ -101,6 +102,7 @@ pub fn setup( subscriber.with(logging_layer).init(); } config::LogFormat::Json => { + error_stack::Report::set_color_mode(error_stack::fmt::ColorMode::None); let logging_layer = FormattingLayer::new(service_name, console_writer).with_filter(console_filter); subscriber.with(logging_layer).init(); @@ -131,6 +133,92 @@ fn get_opentelemetry_exporter(config: &config::LogTelemetry) -> TonicExporterBui exporter_builder } +#[derive(Debug, Clone)] +enum TraceUrlAssert { + Match(String), + EndsWith(String), +} + +impl TraceUrlAssert { + fn compare_url(&self, url: &str) -> bool { + match self { + Self::Match(value) => url == value, + Self::EndsWith(end) => url.ends_with(end), + } + } +} + +impl From for TraceUrlAssert { + fn from(value: String) -> Self { + match value { + url if url.starts_with('*') => Self::EndsWith(url.trim_start_matches('*').to_string()), + url => Self::Match(url), + } + } +} + +#[derive(Debug, Clone)] +struct TraceAssertion { + clauses: Option>, + /// default behaviour for tracing if no condition is provided + default: bool, +} + +impl TraceAssertion { + /// + /// Should the provided url be traced + /// + fn should_trace_url(&self, url: &str) -> bool { + match &self.clauses { + Some(clauses) => clauses.iter().all(|cur| cur.compare_url(url)), + None => self.default, + } + } +} + +/// +/// Conditional Sampler for providing control on url based tracing +/// +#[derive(Clone, Debug)] +struct ConditionalSampler(TraceAssertion, T); + +impl trace::ShouldSample for ConditionalSampler { + fn should_sample( + &self, + parent_context: Option<&opentelemetry::Context>, + trace_id: opentelemetry::trace::TraceId, + name: &str, + span_kind: &opentelemetry::trace::SpanKind, + attributes: &opentelemetry::trace::OrderMap, + links: &[opentelemetry::trace::Link], + instrumentation_library: &opentelemetry::InstrumentationLibrary, + ) -> opentelemetry::trace::SamplingResult { + match attributes + .get(&opentelemetry::Key::new("http.route")) + .map_or(self.0.default, |inner| { + self.0.should_trace_url(&inner.as_str()) + }) { + true => self.1.should_sample( + parent_context, + trace_id, + name, + span_kind, + attributes, + links, + instrumentation_library, + ), + false => opentelemetry::trace::SamplingResult { + decision: opentelemetry::trace::SamplingDecision::Drop, + attributes: Vec::new(), + trace_state: match parent_context { + Some(ctx) => ctx.span().span_context().trace_state().clone(), + None => TraceState::default(), + }, + }, + } + } +} + fn setup_tracing_pipeline( config: &config::LogTelemetry, service_name: &str, @@ -139,9 +227,16 @@ fn setup_tracing_pipeline( global::set_text_map_propagator(TraceContextPropagator::new()); let mut trace_config = trace::config() - .with_sampler(trace::Sampler::TraceIdRatioBased( - config.sampling_rate.unwrap_or(1.0), - )) + .with_sampler(trace::Sampler::ParentBased(Box::new(ConditionalSampler( + TraceAssertion { + clauses: config + .route_to_trace + .clone() + .map(|inner| inner.into_iter().map(Into::into).collect()), + default: false, + }, + trace::Sampler::TraceIdRatioBased(config.sampling_rate.unwrap_or(1.0)), + )))) .with_resource(Resource::new(vec![KeyValue::new( "service.name", service_name.to_owned(), diff --git a/crates/scheduler/src/utils.rs b/crates/scheduler/src/utils.rs index 676ef330f9d1..53f14bd1fb9c 100644 --- a/crates/scheduler/src/utils.rs +++ b/crates/scheduler/src/utils.rs @@ -298,6 +298,7 @@ pub fn get_schedule_time( None => mapping.default_mapping, }; + // For first try, get the `start_after` time if retry_count == 0 { Some(mapping.start_after) } else { @@ -328,6 +329,7 @@ pub fn get_pm_schedule_time( } } +/// Get the delay based on the retry count fn get_delay<'a>( retry_count: i32, mut array: impl Iterator, diff --git a/crates/storage_impl/Cargo.toml b/crates/storage_impl/Cargo.toml index 2226baa5c3bd..e4c41c41b6f4 100644 --- a/crates/storage_impl/Cargo.toml +++ b/crates/storage_impl/Cargo.toml @@ -24,6 +24,7 @@ masking = { version = "0.1.0", path = "../masking" } redis_interface = { version = "0.1.0", path = "../redis_interface" } router_env = { version = "0.1.0", path = "../router_env" } external_services = { version = "0.1.0", path = "../external_services" } +router_derive = { version = "0.1.0", path = "../router_derive" } # Third party crates actix-web = "4.3.1" diff --git a/crates/storage_impl/src/config.rs b/crates/storage_impl/src/config.rs index ad543b1942a3..ceed3da81b39 100644 --- a/crates/storage_impl/src/config.rs +++ b/crates/storage_impl/src/config.rs @@ -9,4 +9,5 @@ pub struct Database { pub dbname: String, pub pool_size: u32, pub connection_timeout: u64, + pub queue_strategy: bb8::QueueStrategy, } diff --git a/crates/storage_impl/src/consts.rs b/crates/storage_impl/src/consts.rs new file mode 100644 index 000000000000..04eab6176f94 --- /dev/null +++ b/crates/storage_impl/src/consts.rs @@ -0,0 +1,2 @@ +// TTL for KV setup +pub(crate) const KV_TTL: u32 = 300; diff --git a/crates/storage_impl/src/database/store.rs b/crates/storage_impl/src/database/store.rs index f9a7450b1cd4..a09f1b752561 100644 --- a/crates/storage_impl/src/database/store.rs +++ b/crates/storage_impl/src/database/store.rs @@ -88,6 +88,7 @@ pub async fn diesel_make_pg_pool( let manager = async_bb8_diesel::ConnectionManager::::new(database_url); let mut pool = bb8::Pool::builder() .max_size(database.pool_size) + .queue_strategy(database.queue_strategy) .connection_timeout(std::time::Duration::from_secs(database.connection_timeout)); if test_transaction { diff --git a/crates/storage_impl/src/lib.rs b/crates/storage_impl/src/lib.rs index 5f355e713487..58f4d4956fae 100644 --- a/crates/storage_impl/src/lib.rs +++ b/crates/storage_impl/src/lib.rs @@ -9,6 +9,7 @@ mod address; pub mod config; pub mod connection; mod connector_response; +mod consts; pub mod database; pub mod errors; mod lookup; @@ -17,6 +18,7 @@ pub mod mock_db; pub mod payments; pub mod redis; pub mod refund; +mod reverse_lookup; mod utils; use database::store::PgPool; diff --git a/crates/storage_impl/src/lookup.rs b/crates/storage_impl/src/lookup.rs index 7c425c780670..ae4d1f733903 100644 --- a/crates/storage_impl/src/lookup.rs +++ b/crates/storage_impl/src/lookup.rs @@ -1,21 +1,32 @@ use common_utils::errors::CustomResult; use data_models::errors; -use diesel_models::reverse_lookup::{ - ReverseLookup as DieselReverseLookup, ReverseLookupNew as DieselReverseLookupNew, +use diesel_models::{ + kv, + reverse_lookup::{ + ReverseLookup as DieselReverseLookup, ReverseLookupNew as DieselReverseLookupNew, + }, }; use error_stack::{IntoReport, ResultExt}; +use redis_interface::SetnxReply; -use crate::{redis::cache::get_or_populate_redis, DatabaseStore, KVRouterStore, RouterStore}; +use crate::{ + diesel_error_to_data_error, + redis::kv_store::{kv_wrapper, KvOperation, PartitionKey}, + utils::{self, try_redis_get_else_try_database_get}, + DatabaseStore, KVRouterStore, RouterStore, +}; #[async_trait::async_trait] pub trait ReverseLookupInterface { async fn insert_reverse_lookup( &self, _new: DieselReverseLookupNew, + storage_scheme: data_models::MerchantStorageScheme, ) -> CustomResult; async fn get_lookup_by_lookup_id( &self, _id: &str, + storage_scheme: data_models::MerchantStorageScheme, ) -> CustomResult; } @@ -24,6 +35,7 @@ impl ReverseLookupInterface for RouterStore { async fn insert_reverse_lookup( &self, new: DieselReverseLookupNew, + _storage_scheme: data_models::MerchantStorageScheme, ) -> CustomResult { let conn = self .get_master_pool() @@ -32,7 +44,7 @@ impl ReverseLookupInterface for RouterStore { .into_report() .change_context(errors::StorageError::DatabaseConnectionError)?; new.insert(&conn).await.map_err(|er| { - let new_err = crate::diesel_error_to_data_error(er.current_context()); + let new_err = diesel_error_to_data_error(er.current_context()); er.change_context(new_err) }) } @@ -40,17 +52,15 @@ impl ReverseLookupInterface for RouterStore { async fn get_lookup_by_lookup_id( &self, id: &str, + _storage_scheme: data_models::MerchantStorageScheme, ) -> CustomResult { - let database_call = || async { - let conn = crate::utils::pg_connection_read(self).await?; - DieselReverseLookup::find_by_lookup_id(id, &conn) - .await - .map_err(|er| { - let new_err = crate::diesel_error_to_data_error(er.current_context()); - er.change_context(new_err) - }) - }; - get_or_populate_redis(self, format!("reverse_lookup_{id}"), database_call).await + let conn = utils::pg_connection_read(self).await?; + DieselReverseLookup::find_by_lookup_id(id, &conn) + .await + .map_err(|er| { + let new_err = diesel_error_to_data_error(er.current_context()); + er.change_context(new_err) + }) } } @@ -59,14 +69,82 @@ impl ReverseLookupInterface for KVRouterStore { async fn insert_reverse_lookup( &self, new: DieselReverseLookupNew, + storage_scheme: data_models::MerchantStorageScheme, ) -> CustomResult { - self.router_store.insert_reverse_lookup(new).await + match storage_scheme { + data_models::MerchantStorageScheme::PostgresOnly => { + self.router_store + .insert_reverse_lookup(new, storage_scheme) + .await + } + data_models::MerchantStorageScheme::RedisKv => { + let created_rev_lookup = DieselReverseLookup { + lookup_id: new.lookup_id.clone(), + sk_id: new.sk_id.clone(), + pk_id: new.pk_id.clone(), + source: new.source.clone(), + }; + let combination = &created_rev_lookup.pk_id; + match kv_wrapper::( + self, + KvOperation::SetNx(&created_rev_lookup), + format!("reverse_lookup_{}", &created_rev_lookup.lookup_id), + ) + .await + .change_context(errors::StorageError::KVError)? + .try_into_setnx() + { + Ok(SetnxReply::KeySet) => { + let redis_entry = kv::TypedSql { + op: kv::DBOperation::Insert { + insertable: kv::Insertable::ReverseLookUp(new), + }, + }; + self.push_to_drainer_stream::( + redis_entry, + PartitionKey::MerchantIdPaymentIdCombination { combination }, + ) + .await + .change_context(errors::StorageError::KVError)?; + + Ok(created_rev_lookup) + } + Ok(SetnxReply::KeyNotSet) => Err(errors::StorageError::DuplicateValue { + entity: "reverse_lookup", + key: Some(created_rev_lookup.lookup_id.clone()), + }) + .into_report(), + Err(er) => Err(er).change_context(errors::StorageError::KVError), + } + } + } } async fn get_lookup_by_lookup_id( &self, id: &str, + storage_scheme: data_models::MerchantStorageScheme, ) -> CustomResult { - self.router_store.get_lookup_by_lookup_id(id).await + let database_call = || async { + self.router_store + .get_lookup_by_lookup_id(id, storage_scheme) + .await + }; + match storage_scheme { + data_models::MerchantStorageScheme::PostgresOnly => database_call().await, + data_models::MerchantStorageScheme::RedisKv => { + let redis_fut = async { + kv_wrapper( + self, + KvOperation::::Get, + format!("reverse_lookup_{id}"), + ) + .await? + .try_into_get() + }; + + try_redis_get_else_try_database_get(redis_fut, database_call).await + } + } } } diff --git a/crates/storage_impl/src/payments/payment_attempt.rs b/crates/storage_impl/src/payments/payment_attempt.rs index e4d3973918e9..4764ce68a314 100644 --- a/crates/storage_impl/src/payments/payment_attempt.rs +++ b/crates/storage_impl/src/payments/payment_attempt.rs @@ -28,7 +28,7 @@ use router_env::{instrument, tracing}; use crate::{ diesel_error_to_data_error, lookup::ReverseLookupInterface, - redis::kv_store::{PartitionKey, RedisConnInterface}, + redis::kv_store::{kv_wrapper, KvOperation, PartitionKey}, utils::{pg_connection_read, pg_connection_write, try_redis_get_else_try_database_get}, DataModelExt, DatabaseStore, KVRouterStore, RouterStore, }; @@ -308,7 +308,7 @@ impl PaymentAttemptInterface for KVRouterStore { } MerchantStorageScheme::RedisKv => { let key = format!( - "{}_{}", + "mid_{}_pid_{}", payment_attempt.merchant_id, payment_attempt.payment_id ); @@ -362,14 +362,15 @@ impl PaymentAttemptInterface for KVRouterStore { }; let field = format!("pa_{}", created_attempt.attempt_id); - match self - .get_redis_conn() - .map_err(|er| { - let error = format!("{}", er); - er.change_context(errors::StorageError::RedisError(error)) - })? - .serialize_and_set_hash_field_if_not_exist(&key, &field, &created_attempt) - .await + + match kv_wrapper::( + self, + KvOperation::HSetNx(&field, &created_attempt), + &key, + ) + .await + .change_context(errors::StorageError::KVError)? + .try_into_hsetnx() { Ok(HsetnxReply::KeyNotSet) => Err(errors::StorageError::DuplicateValue { entity: "payment attempt", @@ -377,10 +378,8 @@ impl PaymentAttemptInterface for KVRouterStore { }) .into_report(), Ok(HsetnxReply::KeySet) => { - let conn = pg_connection_write(self).await?; - //Reverse lookup for attempt_id - ReverseLookupNew { + let reverse_lookup = ReverseLookupNew { lookup_id: format!( "{}_{}", &created_attempt.merchant_id, &created_attempt.attempt_id, @@ -388,13 +387,9 @@ impl PaymentAttemptInterface for KVRouterStore { pk_id: key, sk_id: field, source: "payment_attempt".to_string(), - } - .insert(&conn) - .await - .map_err(|er| { - let new_err = diesel_error_to_data_error(er.current_context()); - er.change_context(new_err) - })?; + }; + self.insert_reverse_lookup(reverse_lookup, storage_scheme) + .await?; let redis_entry = kv::TypedSql { op: kv::DBOperation::Insert { @@ -434,7 +429,7 @@ impl PaymentAttemptInterface for KVRouterStore { .await } MerchantStorageScheme::RedisKv => { - let key = format!("{}_{}", this.merchant_id, this.payment_id); + let key = format!("mid_{}_pid_{}", this.merchant_id, this.payment_id); let old_connector_transaction_id = &this.connector_transaction_id; let old_preprocessing_id = &this.preprocessing_step_id; let updated_attempt = PaymentAttempt::from_storage_model( @@ -448,16 +443,16 @@ impl PaymentAttemptInterface for KVRouterStore { .into_report() .change_context(errors::StorageError::KVError)?; let field = format!("pa_{}", updated_attempt.attempt_id); - let updated_attempt = self - .get_redis_conn() - .map_err(|er| { - let error = format!("{}", er); - er.change_context(errors::StorageError::RedisError(error)) - })? - .set_hash_fields(&key, (&field, &redis_value)) - .await - .map(|_| updated_attempt) - .change_context(errors::StorageError::KVError)?; + + kv_wrapper::<(), _, _>( + self, + KvOperation::Hset::((&field, redis_value)), + &key, + ) + .await + .change_context(errors::StorageError::KVError)? + .try_into_hset() + .change_context(errors::StorageError::KVError)?; match ( old_connector_transaction_id, @@ -465,22 +460,24 @@ impl PaymentAttemptInterface for KVRouterStore { ) { (None, Some(connector_transaction_id)) => { add_connector_txn_id_to_reverse_lookup( - &self.router_store, + self, key.as_str(), this.merchant_id.as_str(), updated_attempt.attempt_id.as_str(), connector_transaction_id.as_str(), + storage_scheme, ) .await?; } (Some(old_connector_transaction_id), Some(connector_transaction_id)) => { if old_connector_transaction_id.ne(connector_transaction_id) { add_connector_txn_id_to_reverse_lookup( - &self.router_store, + self, key.as_str(), this.merchant_id.as_str(), updated_attempt.attempt_id.as_str(), connector_transaction_id.as_str(), + storage_scheme, ) .await?; } @@ -491,22 +488,24 @@ impl PaymentAttemptInterface for KVRouterStore { match (old_preprocessing_id, &updated_attempt.preprocessing_step_id) { (None, Some(preprocessing_id)) => { add_preprocessing_id_to_reverse_lookup( - &self.router_store, + self, key.as_str(), this.merchant_id.as_str(), updated_attempt.attempt_id.as_str(), preprocessing_id.as_str(), + storage_scheme, ) .await?; } (Some(old_preprocessing_id), Some(preprocessing_id)) => { if old_preprocessing_id.ne(preprocessing_id) { add_preprocessing_id_to_reverse_lookup( - &self.router_store, + self, key.as_str(), this.merchant_id.as_str(), updated_attempt.attempt_id.as_str(), preprocessing_id.as_str(), + storage_scheme, ) .await?; } @@ -559,16 +558,15 @@ impl PaymentAttemptInterface for KVRouterStore { MerchantStorageScheme::RedisKv => { // We assume that PaymentAttempt <=> PaymentIntent is a one-to-one relation for now let lookup_id = format!("{merchant_id}_{connector_transaction_id}"); - let lookup = self.get_lookup_by_lookup_id(&lookup_id).await?; + let lookup = self + .get_lookup_by_lookup_id(&lookup_id, storage_scheme) + .await?; let key = &lookup.pk_id; try_redis_get_else_try_database_get( - self.get_redis_conn() - .map_err(|er| { - let error = format!("{}", er); - er.change_context(errors::StorageError::RedisError(error)) - })? - .get_hash_field_and_deserialize(key, &lookup.sk_id, "PaymentAttempt"), + async { + kv_wrapper(self, KvOperation::::HGet(&lookup.sk_id), key).await?.try_into_hget() + }, || async {self.router_store.find_payment_attempt_by_connector_transaction_id_payment_id_merchant_id(connector_transaction_id, payment_id, merchant_id, storage_scheme).await}, ) .await @@ -593,26 +591,27 @@ impl PaymentAttemptInterface for KVRouterStore { match storage_scheme { MerchantStorageScheme::PostgresOnly => database_call().await, MerchantStorageScheme::RedisKv => { - let key = format!("{merchant_id}_{payment_id}"); + let key = format!("mid_{merchant_id}_pid_{payment_id}"); let pattern = "pa_*"; - let redis_conn = self - .get_redis_conn() - .change_context(errors::StorageError::KVError)?; let redis_fut = async { - redis_conn - .hscan_and_deserialize::(&key, pattern, None) - .await - .and_then(|mut payment_attempts| { - payment_attempts.sort_by(|a, b| b.modified_at.cmp(&a.modified_at)); - payment_attempts - .iter() - .find(|&pa| pa.status == api_models::enums::AttemptStatus::Charged) - .cloned() - .ok_or(error_stack::report!( - redis_interface::errors::RedisError::NotFound - )) - }) + let kv_result = kv_wrapper::( + self, + KvOperation::::Scan(pattern), + key, + ) + .await? + .try_into_scan(); + kv_result.and_then(|mut payment_attempts| { + payment_attempts.sort_by(|a, b| b.modified_at.cmp(&a.modified_at)); + payment_attempts + .iter() + .find(|&pa| pa.status == api_models::enums::AttemptStatus::Charged) + .cloned() + .ok_or(error_stack::report!( + redis_interface::errors::RedisError::NotFound + )) + }) }; try_redis_get_else_try_database_get(redis_fut, database_call).await } @@ -637,16 +636,21 @@ impl PaymentAttemptInterface for KVRouterStore { } MerchantStorageScheme::RedisKv => { let lookup_id = format!("{merchant_id}_{connector_txn_id}"); - let lookup = self.get_lookup_by_lookup_id(&lookup_id).await?; + let lookup = self + .get_lookup_by_lookup_id(&lookup_id, storage_scheme) + .await?; let key = &lookup.pk_id; try_redis_get_else_try_database_get( - self.get_redis_conn() - .map_err(|er| { - let error = format!("{}", er); - er.change_context(errors::StorageError::RedisError(error)) - })? - .get_hash_field_and_deserialize(key, &lookup.sk_id, "PaymentAttempt"), + async { + kv_wrapper( + self, + KvOperation::::HGet(&lookup.sk_id), + key, + ) + .await? + .try_into_hget() + }, || async { self.router_store .find_payment_attempt_by_merchant_id_connector_txn_id( @@ -682,15 +686,14 @@ impl PaymentAttemptInterface for KVRouterStore { .await } MerchantStorageScheme::RedisKv => { - let key = format!("{merchant_id}_{payment_id}"); + let key = format!("mid_{merchant_id}_pid_{payment_id}"); let field = format!("pa_{attempt_id}"); try_redis_get_else_try_database_get( - self.get_redis_conn() - .map_err(|er| { - let error = format!("{}", er); - er.change_context(errors::StorageError::RedisError(error)) - })? - .get_hash_field_and_deserialize(&key, &field, "PaymentAttempt"), + async { + kv_wrapper(self, KvOperation::::HGet(&field), key) + .await? + .try_into_hget() + }, || async { self.router_store .find_payment_attempt_by_payment_id_merchant_id_attempt_id( @@ -725,15 +728,20 @@ impl PaymentAttemptInterface for KVRouterStore { } MerchantStorageScheme::RedisKv => { let lookup_id = format!("{merchant_id}_{attempt_id}"); - let lookup = self.get_lookup_by_lookup_id(&lookup_id).await?; + let lookup = self + .get_lookup_by_lookup_id(&lookup_id, storage_scheme) + .await?; let key = &lookup.pk_id; try_redis_get_else_try_database_get( - self.get_redis_conn() - .map_err(|er| { - let error = format!("{}", er); - er.change_context(errors::StorageError::RedisError(error)) - })? - .get_hash_field_and_deserialize(key, &lookup.sk_id, "PaymentAttempt"), + async { + kv_wrapper( + self, + KvOperation::::HGet(&lookup.sk_id), + key, + ) + .await? + .try_into_hget() + }, || async { self.router_store .find_payment_attempt_by_attempt_id_merchant_id( @@ -767,16 +775,21 @@ impl PaymentAttemptInterface for KVRouterStore { } MerchantStorageScheme::RedisKv => { let lookup_id = format!("{merchant_id}_{preprocessing_id}"); - let lookup = self.get_lookup_by_lookup_id(&lookup_id).await?; + let lookup = self + .get_lookup_by_lookup_id(&lookup_id, storage_scheme) + .await?; let key = &lookup.pk_id; try_redis_get_else_try_database_get( - self.get_redis_conn() - .map_err(|er| { - let error = format!("{}", er); - er.change_context(errors::StorageError::RedisError(error)) - })? - .get_hash_field_and_deserialize(key, &lookup.sk_id, "PaymentAttempt"), + async { + kv_wrapper( + self, + KvOperation::::HGet(&lookup.sk_id), + key, + ) + .await? + .try_into_hget() + }, || async { self.router_store .find_payment_attempt_by_preprocessing_id_merchant_id( @@ -809,15 +822,12 @@ impl PaymentAttemptInterface for KVRouterStore { .await } MerchantStorageScheme::RedisKv => { - let key = format!("{merchant_id}_{payment_id}"); + let key = format!("mid_{merchant_id}_pid_{payment_id}"); - self.get_redis_conn() - .map_err(|er| { - let error = format!("{}", er); - er.change_context(errors::StorageError::RedisError(error)) - })? - .hscan_and_deserialize(&key, "pa_*", None) + kv_wrapper(self, KvOperation::::Scan("pa_*"), key) .await + .change_context(errors::StorageError::KVError)? + .try_into_scan() .change_context(errors::StorageError::KVError) } } @@ -1496,48 +1506,42 @@ impl DataModelExt for PaymentAttemptUpdate { #[inline] async fn add_connector_txn_id_to_reverse_lookup( - store: &RouterStore, + store: &KVRouterStore, key: &str, merchant_id: &str, updated_attempt_attempt_id: &str, connector_transaction_id: &str, + storage_scheme: MerchantStorageScheme, ) -> CustomResult { - let conn = pg_connection_write(store).await?; let field = format!("pa_{}", updated_attempt_attempt_id); - ReverseLookupNew { + let reverse_lookup_new = ReverseLookupNew { lookup_id: format!("{}_{}", merchant_id, connector_transaction_id), pk_id: key.to_owned(), sk_id: field.clone(), source: "payment_attempt".to_string(), - } - .insert(&conn) - .await - .map_err(|err| { - let new_err = diesel_error_to_data_error(err.current_context()); - err.change_context(new_err) - }) + }; + store + .insert_reverse_lookup(reverse_lookup_new, storage_scheme) + .await } #[inline] async fn add_preprocessing_id_to_reverse_lookup( - store: &RouterStore, + store: &KVRouterStore, key: &str, merchant_id: &str, updated_attempt_attempt_id: &str, preprocessing_id: &str, + storage_scheme: MerchantStorageScheme, ) -> CustomResult { - let conn = pg_connection_write(store).await?; let field = format!("pa_{}", updated_attempt_attempt_id); - ReverseLookupNew { + let reverse_lookup_new = ReverseLookupNew { lookup_id: format!("{}_{}", merchant_id, preprocessing_id), pk_id: key.to_owned(), sk_id: field.clone(), source: "payment_attempt".to_string(), - } - .insert(&conn) - .await - .map_err(|er| { - let new_err = diesel_error_to_data_error(er.current_context()); - er.change_context(new_err) - }) + }; + store + .insert_reverse_lookup(reverse_lookup_new, storage_scheme) + .await } diff --git a/crates/storage_impl/src/payments/payment_intent.rs b/crates/storage_impl/src/payments/payment_intent.rs index 8352158be052..6814512028b4 100644 --- a/crates/storage_impl/src/payments/payment_intent.rs +++ b/crates/storage_impl/src/payments/payment_intent.rs @@ -35,7 +35,7 @@ use router_env::logger; use router_env::{instrument, tracing}; use crate::{ - redis::kv_store::{PartitionKey, RedisConnInterface}, + redis::kv_store::{kv_wrapper, KvOperation, PartitionKey}, utils::{pg_connection_read, pg_connection_write}, DataModelExt, DatabaseStore, KVRouterStore, }; @@ -55,7 +55,7 @@ impl PaymentIntentInterface for KVRouterStore { } MerchantStorageScheme::RedisKv => { - let key = format!("{}_{}", new.merchant_id, new.payment_id); + let key = format!("mid_{}_pid_{}", new.merchant_id, new.payment_id); let field = format!("pi_{}", new.payment_id); let created_intent = PaymentIntent { id: 0i32, @@ -93,11 +93,14 @@ impl PaymentIntentInterface for KVRouterStore { payment_confirm_source: new.payment_confirm_source, }; - match self - .get_redis_conn() - .change_context(StorageError::DatabaseConnectionError)? - .serialize_and_set_hash_field_if_not_exist(&key, &field, &created_intent) - .await + match kv_wrapper::( + self, + KvOperation::HSetNx(&field, &created_intent), + &key, + ) + .await + .change_context(StorageError::KVError)? + .try_into_hsetnx() { Ok(HsetnxReply::KeyNotSet) => Err(StorageError::DuplicateValue { entity: "payment_intent", @@ -141,7 +144,7 @@ impl PaymentIntentInterface for KVRouterStore { .await } MerchantStorageScheme::RedisKv => { - let key = format!("{}_{}", this.merchant_id, this.payment_id); + let key = format!("mid_{}_pid_{}", this.merchant_id, this.payment_id); let field = format!("pi_{}", this.payment_id); let updated_intent = payment_intent.clone().apply_changeset(this.clone()); @@ -151,13 +154,15 @@ impl PaymentIntentInterface for KVRouterStore { Encode::::encode_to_string_of_json(&updated_intent) .change_context(StorageError::SerializationFailed)?; - let updated_intent = self - .get_redis_conn() - .change_context(StorageError::DatabaseConnectionError)? - .set_hash_fields(&key, (&field, &redis_value)) - .await - .map(|_| updated_intent) - .change_context(StorageError::KVError)?; + kv_wrapper::<(), _, _>( + self, + KvOperation::::Hset((&field, redis_value)), + &key, + ) + .await + .change_context(StorageError::KVError)? + .try_into_hset() + .change_context(StorageError::KVError)?; let redis_entry = kv::TypedSql { op: kv::DBOperation::Update { @@ -204,12 +209,18 @@ impl PaymentIntentInterface for KVRouterStore { MerchantStorageScheme::PostgresOnly => database_call().await, MerchantStorageScheme::RedisKv => { - let key = format!("{merchant_id}_{payment_id}"); + let key = format!("mid_{merchant_id}_pid_{payment_id}"); let field = format!("pi_{payment_id}"); crate::utils::try_redis_get_else_try_database_get( - self.get_redis_conn() - .change_context(StorageError::DatabaseConnectionError)? - .get_hash_field_and_deserialize(&key, &field, "PaymentIntent"), + async { + kv_wrapper::( + self, + KvOperation::::HGet(&field), + &key, + ) + .await? + .try_into_hget() + }, database_call, ) .await @@ -224,15 +235,11 @@ impl PaymentIntentInterface for KVRouterStore { filters: &PaymentIntentFetchConstraints, storage_scheme: MerchantStorageScheme, ) -> error_stack::Result, StorageError> { - match storage_scheme { - MerchantStorageScheme::PostgresOnly => { - self.router_store - .filter_payment_intent_by_constraints(merchant_id, filters, storage_scheme) - .await - } - MerchantStorageScheme::RedisKv => Err(StorageError::KVError.into()), - } + self.router_store + .filter_payment_intent_by_constraints(merchant_id, filters, storage_scheme) + .await } + #[cfg(feature = "olap")] async fn filter_payment_intents_by_time_range_constraints( &self, @@ -240,18 +247,13 @@ impl PaymentIntentInterface for KVRouterStore { time_range: &api_models::payments::TimeRange, storage_scheme: MerchantStorageScheme, ) -> error_stack::Result, StorageError> { - match storage_scheme { - MerchantStorageScheme::PostgresOnly => { - self.router_store - .filter_payment_intents_by_time_range_constraints( - merchant_id, - time_range, - storage_scheme, - ) - .await - } - MerchantStorageScheme::RedisKv => Err(StorageError::KVError.into()), - } + self.router_store + .filter_payment_intents_by_time_range_constraints( + merchant_id, + time_range, + storage_scheme, + ) + .await } #[cfg(feature = "olap")] @@ -261,14 +263,9 @@ impl PaymentIntentInterface for KVRouterStore { filters: &PaymentIntentFetchConstraints, storage_scheme: MerchantStorageScheme, ) -> error_stack::Result, StorageError> { - match storage_scheme { - MerchantStorageScheme::PostgresOnly => { - self.router_store - .get_filtered_payment_intents_attempt(merchant_id, filters, storage_scheme) - .await - } - MerchantStorageScheme::RedisKv => Err(StorageError::KVError.into()), - } + self.router_store + .get_filtered_payment_intents_attempt(merchant_id, filters, storage_scheme) + .await } #[cfg(feature = "olap")] @@ -278,19 +275,13 @@ impl PaymentIntentInterface for KVRouterStore { constraints: &PaymentIntentFetchConstraints, storage_scheme: MerchantStorageScheme, ) -> error_stack::Result, StorageError> { - match storage_scheme { - MerchantStorageScheme::PostgresOnly => { - self.router_store - .get_filtered_active_attempt_ids_for_total_count( - merchant_id, - constraints, - storage_scheme, - ) - .await - } - - MerchantStorageScheme::RedisKv => Err(StorageError::KVError.into()), - } + self.router_store + .get_filtered_active_attempt_ids_for_total_count( + merchant_id, + constraints, + storage_scheme, + ) + .await } } diff --git a/crates/storage_impl/src/redis/kv_store.rs b/crates/storage_impl/src/redis/kv_store.rs index 407cb838d862..e9600d04de52 100644 --- a/crates/storage_impl/src/redis/kv_store.rs +++ b/crates/storage_impl/src/redis/kv_store.rs @@ -1,4 +1,11 @@ -use std::sync::Arc; +use std::{fmt::Debug, sync::Arc}; + +use common_utils::errors::CustomResult; +use redis_interface::errors::RedisError; +use router_derive::TryGetEnumVariant; +use serde::de; + +use crate::{consts, KVRouterStore}; pub trait KvStorePartition { fn partition_number(key: PartitionKey<'_>, num_partitions: u8) -> u32 { @@ -16,6 +23,9 @@ pub enum PartitionKey<'a> { merchant_id: &'a str, payment_id: &'a str, }, + MerchantIdPaymentIdCombination { + combination: &'a str, + }, } impl<'a> std::fmt::Display for PartitionKey<'a> { @@ -25,6 +35,9 @@ impl<'a> std::fmt::Display for PartitionKey<'a> { merchant_id, payment_id, } => f.write_str(&format!("mid_{merchant_id}_pid_{payment_id}")), + PartitionKey::MerchantIdPaymentIdCombination { combination } => { + f.write_str(combination) + } } } } @@ -32,8 +45,76 @@ impl<'a> std::fmt::Display for PartitionKey<'a> { pub trait RedisConnInterface { fn get_redis_conn( &self, - ) -> error_stack::Result< - Arc, - redis_interface::errors::RedisError, - >; + ) -> error_stack::Result, RedisError>; +} + +pub enum KvOperation<'a, S: serde::Serialize + Debug> { + Hset((&'a str, String)), + SetNx(S), + HSetNx(&'a str, S), + HGet(&'a str), + Get, + Scan(&'a str), +} + +#[derive(TryGetEnumVariant)] +#[error(RedisError(UnknownResult))] +pub enum KvResult { + HGet(T), + Get(T), + Hset(()), + SetNx(redis_interface::SetnxReply), + HSetNx(redis_interface::HsetnxReply), + Scan(Vec), +} + +pub async fn kv_wrapper<'a, T, D, S>( + store: &KVRouterStore, + op: KvOperation<'a, S>, + key: impl AsRef, +) -> CustomResult, RedisError> +where + T: de::DeserializeOwned, + D: crate::database::store::DatabaseStore, + S: serde::Serialize + Debug, +{ + let redis_conn = store.get_redis_conn()?; + + let key = key.as_ref(); + let type_name = std::any::type_name::(); + + match op { + KvOperation::Hset(value) => { + redis_conn + .set_hash_fields(key, value, Some(consts::KV_TTL)) + .await?; + Ok(KvResult::Hset(())) + } + KvOperation::HGet(field) => { + let result = redis_conn + .get_hash_field_and_deserialize(key, field, type_name) + .await?; + Ok(KvResult::HGet(result)) + } + KvOperation::Scan(pattern) => { + let result: Vec = redis_conn.hscan_and_deserialize(key, pattern, None).await?; + Ok(KvResult::Scan(result)) + } + KvOperation::HSetNx(field, value) => { + let result = redis_conn + .serialize_and_set_hash_field_if_not_exist(key, field, value, Some(consts::KV_TTL)) + .await?; + Ok(KvResult::HSetNx(result)) + } + KvOperation::SetNx(value) => { + let result = redis_conn + .serialize_and_set_key_if_not_exist(key, value, Some(consts::KV_TTL.into())) + .await?; + Ok(KvResult::SetNx(result)) + } + KvOperation::Get => { + let result = redis_conn.get_and_deserialize_key(key, type_name).await?; + Ok(KvResult::Get(result)) + } + } } diff --git a/crates/storage_impl/src/reverse_lookup.rs b/crates/storage_impl/src/reverse_lookup.rs new file mode 100644 index 000000000000..9466267a3cee --- /dev/null +++ b/crates/storage_impl/src/reverse_lookup.rs @@ -0,0 +1,5 @@ +use diesel_models::reverse_lookup::ReverseLookup; + +use crate::redis::kv_store::KvStorePartition; + +impl KvStorePartition for ReverseLookup {} diff --git a/crates/test_utils/README.md b/crates/test_utils/README.md index 3cbda63e52a1..bfed3479b3f6 100644 --- a/crates/test_utils/README.md +++ b/crates/test_utils/README.md @@ -2,48 +2,66 @@ The heart of `newman`(with directory support) and `UI-tests` -## Newman Usage +## Newman -- Make sure you that you've _**do not**_ have the official newman installed but rather the `newman` fork with directory support - - `newman` can be installed by running `npm install -g 'https://github.com/knutties/newman.git#feature/newman-dir'` - - To see the features that the fork of `newman` supports, click [_**here**_](https://github.com/knutties/newman/blob/feature/newman-dir/DIR_COMMANDS.md) -- Add the connector credentials to the `connector_auth.toml` / `auth.toml` +- Make sure you that you _**do not**_ have the newman (from the Postman team) installed but rather the `newman` fork with directory support +- The `newman` fork can be installed by running `npm install -g 'https://github.com/knutties/newman.git#feature/newman-dir'` +- To see the features that the fork of `newman` supports, click [_**here**_](https://github.com/knutties/newman/blob/feature/newman-dir/DIR_COMMANDS.md) + +## Test Utils Usage + +- Add the connector credentials to the `connector_auth.toml` / `auth.toml` by creating a copy of the `sample_auth.toml` from `router/tests/connectors/sample_auth.toml` - Export the auth file path as an environment variable: ```shell export CONNECTOR_AUTH_FILE_PATH=/path/to/auth.toml ``` -- Run the tests: + +### Supported Commands + +Required fields: + +- `--admin_api_key` -- Admin API Key of the environment. `test_admin` is the Admin API Key for running locally +- `--base_url` -- Base URL of the environment. `http://127.0.0.1:8080` / `http://localhost:8080` is the Base URL for running locally +- `--connector_name` -- Name of the connector that you wish to run. Example: `adyen`, `shift4`, `stripe` + +Optional fields: + +- `--folder` -- To run individual folders in the collection + - Use double quotes to specify folder name. If you wish to run multiple folders, separate them with a comma (`,`) + - Example: `--folder "QuickStart"` or `--folder "Health check,QuickStart"` +- `--verbose` -- A boolean to print detailed logs (requests and responses) + +**Note:** Passing `--verbose` will also print the connector as well as admin API keys in the logs. So, make sure you don't push the commands with `--verbose` to any public repository. + +### Running tests + +- Tests can be run with the following command: ```shell - cargo run --package test_utils --bin test_utils -- --connector_name= --base_url= --admin_api_key= + cargo run --package test_utils --bin test_utils -- --connector_name= --base_url= --admin_api_key= \ + # optionally + --folder ",,..." --verbose ``` -**Note:** You can optionally pass `--verbose` to see the logs of the tests. But make sure you that passing `--verbose` will also print the API-Keys in the logs. So, make sure you don't push the logs to any public repository. Below is an example: -```shell -cargo run --package test_utils --bin test_utils -- --connector_name= --base_url= --admin_api_key= --verbose -``` - -### Running newman locally +**Note**: You can omit `--package test_utils` at the time of running the above command since it is optional. -Execute the following commands: -```shell -export CONNECTOR_AUTH_FILE_PATH=/path/to/auth.toml -cargo run --package test_utils --bin test_utils -- --connector_name= --base_url=http://127.0.0.1:8080 --admin_api_key=test_admin -# Optionally, you can add `--verbose` in the end -``` ## UI tests To run the UI tests, run the following command: + ```shell cargo test --package test_utils --test connectors -- :: --test-threads=1 ``` + ### Example Below is an example to run UI test to only run the `GooglePay` payment test for `adyen` connector: + ```shell cargo test --package test_utils --test connectors -- adyen_uk_ui::should_make_gpay_payment_test --test-threads=1 ``` Below is an example to run all the UI tests for `adyen` connector: + ```shell cargo test --package test_utils --test connectors -- adyen_uk_ui:: --test-threads=1 -``` \ No newline at end of file +``` diff --git a/crates/test_utils/src/lib.rs b/crates/test_utils/src/lib.rs index 82b4addfb942..f7cde3667d94 100644 --- a/crates/test_utils/src/lib.rs +++ b/crates/test_utils/src/lib.rs @@ -1 +1,2 @@ pub mod connector_auth; +pub mod newman_runner; diff --git a/crates/test_utils/src/main.rs b/crates/test_utils/src/main.rs index 155870ee2cb4..637122e468e6 100644 --- a/crates/test_utils/src/main.rs +++ b/crates/test_utils/src/main.rs @@ -1,141 +1,9 @@ -use std::{ - env, - process::{exit, Command}, -}; +use std::process::{exit, Command}; -use clap::{arg, command, Parser}; -use masking::PeekInterface; -use test_utils::connector_auth::{ConnectorAuthType, ConnectorAuthenticationMap}; - -// Just by the name of the connector, this function generates the name of the collection dir -// Example: CONNECTOR_NAME="stripe" -> OUTPUT: postman/collection-dir/stripe -#[inline] -fn get_path(name: impl AsRef) -> String { - format!("postman/collection-dir/{}", name.as_ref()) -} - -#[derive(Parser)] -#[command(version, about = "Postman collection runner using newman!", long_about = None)] -struct Args { - /// Name of the connector - #[arg(short, long = "connector_name")] - connector_name: String, - /// Base URL of the Hyperswitch environment - #[arg(short, long = "base_url")] - base_url: String, - /// Admin API Key of the environment - #[arg(short, long = "admin_api_key")] - admin_api_key: String, - /// Optional Verbose logs - #[arg(short, long)] - verbose: bool, -} +use test_utils::newman_runner; fn main() { - let args = Args::parse(); - - let connector_name = args.connector_name; - let base_url = args.base_url; - let admin_api_key = args.admin_api_key; - - let collection_path = get_path(&connector_name); - let auth_map = ConnectorAuthenticationMap::new(); - - let inner_map = auth_map.inner(); - - // Newman runner - // Depending on the conditions satisfied, variables are added. Since certificates of stripe have already - // been added to the postman collection, those conditions are set to true and collections that have - // variables set up for certificate, will consider those variables and will fail. - - let mut newman_command = Command::new("newman"); - newman_command.args(["dir-run", &collection_path]); - newman_command.args(["--env-var", &format!("admin_api_key={admin_api_key}")]); - newman_command.args(["--env-var", &format!("baseUrl={base_url}")]); - - if let Some(auth_type) = inner_map.get(&connector_name) { - match auth_type { - ConnectorAuthType::HeaderKey { api_key } => { - // newman_command.args(["--env-var", &format!("connector_api_key={}", api_key.map(|val| val))]); - newman_command.args([ - "--env-var", - &format!("connector_api_key={}", api_key.peek()), - ]); - } - ConnectorAuthType::BodyKey { api_key, key1 } => { - newman_command.args([ - "--env-var", - &format!("connector_api_key={}", api_key.peek()), - "--env-var", - &format!("connector_key1={}", key1.peek()), - ]); - } - ConnectorAuthType::SignatureKey { - api_key, - key1, - api_secret, - } => { - newman_command.args([ - "--env-var", - &format!("connector_api_key={}", api_key.peek()), - "--env-var", - &format!("connector_key1={}", key1.peek()), - "--env-var", - &format!("connector_api_secret={}", api_secret.peek()), - ]); - } - ConnectorAuthType::MultiAuthKey { - api_key, - key1, - key2, - api_secret, - } => { - newman_command.args([ - "--env-var", - &format!("connector_api_key={}", api_key.peek()), - "--env-var", - &format!("connector_key1={}", key1.peek()), - "--env-var", - &format!("connector_key1={}", key2.peek()), - "--env-var", - &format!("connector_api_secret={}", api_secret.peek()), - ]); - } - // Handle other ConnectorAuthType variants - _ => { - eprintln!("Invalid authentication type."); - } - } - } else { - eprintln!("Connector not found."); - } - - // Add additional environment variables if present - if let Ok(gateway_merchant_id) = env::var("GATEWAY_MERCHANT_ID") { - newman_command.args([ - "--env-var", - &format!("gateway_merchant_id={gateway_merchant_id}"), - ]); - } - - if let Ok(gpay_certificate) = env::var("GPAY_CERTIFICATE") { - newman_command.args(["--env-var", &format!("certificate={gpay_certificate}")]); - } - - if let Ok(gpay_certificate_keys) = env::var("GPAY_CERTIFICATE_KEYS") { - newman_command.args([ - "--env-var", - &format!("certificate_keys={gpay_certificate_keys}"), - ]); - } - - newman_command.arg("--delay-request").arg("5"); - - newman_command.arg("--color").arg("on"); - - if args.verbose { - newman_command.arg("--verbose"); - } + let mut newman_command: Command = newman_runner::command_generate(); // Execute the newman command let output = newman_command.spawn(); diff --git a/crates/test_utils/src/newman_runner.rs b/crates/test_utils/src/newman_runner.rs new file mode 100644 index 000000000000..c51556f8f255 --- /dev/null +++ b/crates/test_utils/src/newman_runner.rs @@ -0,0 +1,155 @@ +use std::{env, process::Command}; + +use clap::{arg, command, Parser}; +use masking::PeekInterface; + +use crate::connector_auth::{ConnectorAuthType, ConnectorAuthenticationMap}; + +#[derive(Parser)] +#[command(version, about = "Postman collection runner using newman!", long_about = None)] +struct Args { + /// Admin API Key of the environment + #[arg(short, long = "admin_api_key")] + admin_api_key: String, + /// Base URL of the Hyperswitch environment + #[arg(short, long = "base_url")] + base_url: String, + /// Name of the connector + #[arg(short, long = "connector_name")] + connector_name: String, + /// Folder name of specific tests + #[arg(short, long = "folder")] + folders: Option, + /// Optional Verbose logs + #[arg(short, long)] + verbose: bool, +} + +// Just by the name of the connector, this function generates the name of the collection dir +// Example: CONNECTOR_NAME="stripe" -> OUTPUT: postman/collection-dir/stripe +#[inline] +fn get_path(name: impl AsRef) -> String { + format!("postman/collection-dir/{}", name.as_ref()) +} + +pub fn command_generate() -> Command { + let args = Args::parse(); + + let connector_name = args.connector_name; + let base_url = args.base_url; + let admin_api_key = args.admin_api_key; + + let collection_path = get_path(&connector_name); + let auth_map = ConnectorAuthenticationMap::new(); + + let inner_map = auth_map.inner(); + + // Newman runner + // Depending on the conditions satisfied, variables are added. Since certificates of stripe have already + // been added to the postman collection, those conditions are set to true and collections that have + // variables set up for certificate, will consider those variables and will fail. + + let mut newman_command = Command::new("newman"); + newman_command.args(["dir-run", &collection_path]); + newman_command.args(["--env-var", &format!("admin_api_key={admin_api_key}")]); + newman_command.args(["--env-var", &format!("baseUrl={base_url}")]); + + if let Some(auth_type) = inner_map.get(&connector_name) { + match auth_type { + ConnectorAuthType::HeaderKey { api_key } => { + newman_command.args([ + "--env-var", + &format!("connector_api_key={}", api_key.peek()), + ]); + } + ConnectorAuthType::BodyKey { api_key, key1 } => { + newman_command.args([ + "--env-var", + &format!("connector_api_key={}", api_key.peek()), + "--env-var", + &format!("connector_key1={}", key1.peek()), + ]); + } + ConnectorAuthType::SignatureKey { + api_key, + key1, + api_secret, + } => { + newman_command.args([ + "--env-var", + &format!("connector_api_key={}", api_key.peek()), + "--env-var", + &format!("connector_key1={}", key1.peek()), + "--env-var", + &format!("connector_api_secret={}", api_secret.peek()), + ]); + } + ConnectorAuthType::MultiAuthKey { + api_key, + key1, + key2, + api_secret, + } => { + newman_command.args([ + "--env-var", + &format!("connector_api_key={}", api_key.peek()), + "--env-var", + &format!("connector_key1={}", key1.peek()), + "--env-var", + &format!("connector_key2={}", key2.peek()), + "--env-var", + &format!("connector_api_secret={}", api_secret.peek()), + ]); + } + // Handle other ConnectorAuthType variants + _ => { + eprintln!("Invalid authentication type."); + } + } + } else { + eprintln!("Connector not found."); + } + + // Add additional environment variables if present + if let Ok(gateway_merchant_id) = env::var("GATEWAY_MERCHANT_ID") { + newman_command.args([ + "--env-var", + &format!("gateway_merchant_id={gateway_merchant_id}"), + ]); + } + + if let Ok(gpay_certificate) = env::var("GPAY_CERTIFICATE") { + newman_command.args(["--env-var", &format!("certificate={gpay_certificate}")]); + } + + if let Ok(gpay_certificate_keys) = env::var("GPAY_CERTIFICATE_KEYS") { + newman_command.args([ + "--env-var", + &format!("certificate_keys={gpay_certificate_keys}"), + ]); + } + + newman_command.arg("--delay-request").arg("7"); // 7 milli seconds delay + + newman_command.arg("--color").arg("on"); + + // Add flags for running specific folders + if let Some(folders) = &args.folders { + let folder_names: Vec = folders.split(',').map(|s| s.trim().to_string()).collect(); + + for folder_name in folder_names { + if !folder_name.contains("QuickStart") { + // This is a quick fix, "QuickStart" is intentional to have merchant account and API keys set up + // This will be replaced by a more robust and efficient account creation or reuse existing old account + newman_command.args(["--folder", "QuickStart"]); + } + newman_command.args(["--folder", &folder_name]); + } + } + + if args.verbose { + newman_command.arg("--verbose"); + } + + newman_command +} diff --git a/loadtest/config/development.toml b/loadtest/config/development.toml index db6ed0850c8d..380373049689 100644 --- a/loadtest/config/development.toml +++ b/loadtest/config/development.toml @@ -228,3 +228,5 @@ wallet.paypal = {connector_list = "adyen"} card.credit = {connector_list = "stripe,adyen,authorizedotnet,globalpay,worldpay,multisafepay,nmi,nexinets,noon"} card.debit = {connector_list = "stripe,adyen,authorizedotnet,globalpay,worldpay,multisafepay,nmi,nexinets,noon"} bank_debit.ach = { connector_list = "gocardless"} +bank_debit.becs = { connector_list = "gocardless"} +bank_debit.sepa = { connector_list = "gocardless"}